From 61a1feabe04030c7db950beb949fba153187dc32 Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Thu, 14 May 2026 14:44:24 +0800 Subject: [PATCH 01/19] =?UTF-8?q?feat:=20=E9=9B=86=E6=88=90=20WASM=20?= =?UTF-8?q?=E6=8F=92=E4=BB=B6=EF=BC=88spacegate-plugin-wasm=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 工作区加入 plugin-wasm crate 与 spacegate-plugin-wasm 依赖 - shell 增加 plugin-wasm feature，启动时注册 wasm 插件避免循环依赖 - 二进制增加 wasm feature 开关 - 补充 hai-wasm-demo 与 wasm 示例资源 Co-authored-by: Cursor --- Cargo.toml | 2 + binary/spacegate/Cargo.toml | 1 + crates/plugin-wasm/Cargo.toml | 44 + crates/plugin-wasm/src/abi.rs | 280 +++++ crates/plugin-wasm/src/config.rs | 96 ++ crates/plugin-wasm/src/engine.rs | 24 + crates/plugin-wasm/src/error.rs | 21 + crates/plugin-wasm/src/fetch.rs | 19 + crates/plugin-wasm/src/host_fn.rs | 970 ++++++++++++++++++ crates/plugin-wasm/src/host_state.rs | 158 +++ crates/plugin-wasm/src/lib.rs | 35 + crates/plugin-wasm/src/runtime.rs | 45 + crates/plugin-wasm/src/shell.rs | 96 ++ crates/plugin-wasm/src/vm.rs | 477 +++++++++ crates/shell/Cargo.toml | 2 + crates/shell/src/lib.rs | 3 + resource/hai-wasm-demo/config.json | 5 + .../gateway/hai-demo/config.json | 16 + .../gateway/hai-demo/route/demo.json | 33 + resource/hai-wasm-demo/mock_backends.py | 156 +++ .../hai-wasm-demo/plugin/wasm.hai-mix.json | 18 + resource/wasm/hai_process_mix.wasm | Bin 0 -> 984275 bytes 22 files changed, 2501 insertions(+) create mode 100644 crates/plugin-wasm/Cargo.toml create mode 100644 crates/plugin-wasm/src/abi.rs create mode 100644 crates/plugin-wasm/src/config.rs create mode 100644 crates/plugin-wasm/src/engine.rs create mode 100644 crates/plugin-wasm/src/error.rs create mode 100644 crates/plugin-wasm/src/fetch.rs create mode 100644 crates/plugin-wasm/src/host_fn.rs create mode 100644 crates/plugin-wasm/src/host_state.rs create mode 100644 crates/plugin-wasm/src/lib.rs create mode 100644 crates/plugin-wasm/src/runtime.rs create mode 100644 crates/plugin-wasm/src/shell.rs create mode 100644 crates/plugin-wasm/src/vm.rs create mode 100644 resource/hai-wasm-demo/config.json create mode 100644 resource/hai-wasm-demo/gateway/hai-demo/config.json create mode 100644 resource/hai-wasm-demo/gateway/hai-demo/route/demo.json create mode 100644 resource/hai-wasm-demo/mock_backends.py create mode 100644 resource/hai-wasm-demo/plugin/wasm.hai-mix.json create mode 100755 resource/wasm/hai_process_mix.wasm diff --git a/Cargo.toml b/Cargo.toml index 5a0070dc..378bf01c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/extension/*", "crates/kernel", "crates/plugin", + "crates/plugin-wasm", "crates/model", "crates/config", "crates/shell", @@ -46,6 +47,7 @@ spacegate-plugin = { version = "0.2.0-alpha.4", path = "./crates/plugin" } spacegate-config = { version = "0.2.0-alpha.4", path = "./crates/config" } spacegate-model = { version = "0.2.0-alpha.4", path = "./crates/model" } spacegate-shell = { version = "0.2.0-alpha.4", path = "./crates/shell" } +spacegate-plugin-wasm = { version = "0.2.0-alpha.4", path = "./crates/plugin-wasm" } spacegate-ext-axum = { version = "0.2.0-alpha.4", path = "./crates/extension/axum" } spacegate-ext-redis = { version = "0.2.0-alpha.4", path = "./crates/extension/redis" } diff --git a/binary/spacegate/Cargo.toml b/binary/spacegate/Cargo.toml index 30c6f799..3ac59171 100644 --- a/binary/spacegate/Cargo.toml +++ b/binary/spacegate/Cargo.toml @@ -28,6 +28,7 @@ axum = ["spacegate-shell/ext-axum"] static-openssl = ["openssl/vendored"] dylib = ["spacegate-shell/plugin-dylib"] plugin-all = ["spacegate-shell/plugin-all"] +wasm = ["spacegate-shell/plugin-wasm"] [dependencies] # envy = { } clap = { version = "4.5", features = ["derive", "env"] } diff --git a/crates/plugin-wasm/Cargo.toml b/crates/plugin-wasm/Cargo.toml new file mode 100644 index 00000000..4c4159bd --- /dev/null +++ b/crates/plugin-wasm/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "spacegate-plugin-wasm" +version.workspace = true +authors.workspace = true +description = "Proxy-Wasm host integration for SpaceGate (wasmtime)" +edition.workspace = true +license.workspace = true +repository.workspace = true +readme = "../../../README.md" + +[lib] +name = "spacegate_plugin_wasm" +path = "src/lib.rs" + +[dependencies] +spacegate-plugin = { workspace = true } +spacegate-kernel = { workspace = true } +spacegate-model = { workspace = true } + +wasmtime = { version = "23", default-features = true, features = ["async", "cranelift"] } + +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +serde_yaml = "0.9" + +tracing = { workspace = true } +thiserror = "1" +once_cell = "1.19" + +# host fn: dispatch_http_call 走 reqwest 异步客户端（用 0.12 与 spacegate 的 http=1 对齐） +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } +# 异步驱动 + 待回填 dispatch 状态 +tokio = { workspace = true, features = ["sync", "macros", "rt"] } +# inner.call 拿到的是 hyper Response，host fn 操作 header 时要用 http 类型 +hyper = { workspace = true } +http = "1" +http-body-util = { workspace = true } +bytes = { workspace = true } + +# moka 同步缓存模块（Module 编译产物按 url 缓存） +moka = { version = "0.12", features = ["sync"] } + +[dev-dependencies] +tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/crates/plugin-wasm/src/abi.rs b/crates/plugin-wasm/src/abi.rs new file mode 100644 index 00000000..18d562e6 --- /dev/null +++ b/crates/plugin-wasm/src/abi.rs @@ -0,0 +1,280 @@ + //! proxy-wasm ABI 0.2.x 的基础类型与内存/编码工具。 +//! +//! 主要分三块： +//! 1. `Status` / `Action` / `MapType` / `BufferType` / `StreamType` / `LogLevel` 枚举 +//! 2. `MemoryHelper`：通过 `wasmtime::Memory` 安全读写 guest 线性内存 +//! 3. `pairs`：proxy-wasm 头部 (k, v) 列表的二进制布局编解码 +//! +//! 所有越界访问统一转 `WasmHostError::MemoryOob`，避免 trap 撕裂 Store。 + +use crate::error::WasmHostError; +use wasmtime::{Caller, Memory, StoreContext, StoreContextMut}; + +// ───────────────────────────────────────────────────────── +// 枚举：proxy-wasm 0.2.x ABI（仅列出我们用到的子集） +// ───────────────────────────────────────────────────────── + +/// `proxy_status_t`：所有 host fn 的返回值。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +pub enum Status { + Ok = 0, + NotFound = 1, + BadArgument = 2, + Empty = 7, + InternalFailure = 10, +} + +impl Status { + #[inline] + pub fn as_i32(self) -> i32 { + self as i32 + } +} + +/// `proxy_action_t`：guest 钩子返回。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u32)] +pub enum Action { + Continue = 0, + Pause = 1, +} + +impl Action { + pub fn from_u32(v: u32) -> Self { + match v { + 1 => Action::Pause, + _ => Action::Continue, + } + } +} + +/// `proxy_map_type_t`：头部映射的来源。 +/// +/// 与 proxy-wasm-cpp-host 一致： +/// 0 HttpRequestHeaders / 1 HttpRequestTrailers / +/// 2 HttpResponseHeaders / 3 HttpResponseTrailers / +/// 6 HttpCallResponseHeaders / 7 HttpCallResponseTrailers +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MapType { + HttpRequestHeaders, + HttpRequestTrailers, + HttpResponseHeaders, + HttpResponseTrailers, + HttpCallResponseHeaders, + HttpCallResponseTrailers, + Unknown(i32), +} + +impl MapType { + pub fn from_i32(v: i32) -> Self { + match v { + 0 => MapType::HttpRequestHeaders, + 1 => MapType::HttpRequestTrailers, + 2 => MapType::HttpResponseHeaders, + 3 => MapType::HttpResponseTrailers, + 6 => MapType::HttpCallResponseHeaders, + 7 => MapType::HttpCallResponseTrailers, + other => MapType::Unknown(other), + } + } +} + +/// `proxy_buffer_type_t`：缓冲区来源。 +/// +/// 0 HttpRequestBody / 1 HttpResponseBody / 4 HttpCallResponseBody / +/// 6 VmConfiguration / 7 PluginConfiguration / 8 CallData +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BufferType { + HttpRequestBody, + HttpResponseBody, + HttpCallResponseBody, + VmConfiguration, + PluginConfiguration, + Unknown(i32), +} + +impl BufferType { + pub fn from_i32(v: i32) -> Self { + match v { + 0 => BufferType::HttpRequestBody, + 1 => BufferType::HttpResponseBody, + 4 => BufferType::HttpCallResponseBody, + 6 => BufferType::VmConfiguration, + 7 => BufferType::PluginConfiguration, + other => BufferType::Unknown(other), + } + } +} + +/// `proxy_stream_type_t`：`proxy_continue_stream` / `proxy_close_stream` 参数。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamType { + Request, + Response, + Unknown(i32), +} + +impl StreamType { + pub fn from_i32(v: i32) -> Self { + match v { + 0 => StreamType::Request, + 1 => StreamType::Response, + other => StreamType::Unknown(other), + } + } +} + +/// `proxy_log` 的 level（tracing 转换用）。 +pub fn log_level_to_tracing(level: i32) -> tracing::Level { + match level { + 0 => tracing::Level::TRACE, + 1 => tracing::Level::DEBUG, + 2 => tracing::Level::INFO, + 3 => tracing::Level::WARN, + _ => tracing::Level::ERROR, + } +} + +// ───────────────────────────────────────────────────────── +// MemoryHelper：guest 内存读写（按 host fn 单次调用的生命周期使用） +// ───────────────────────────────────────────────────────── + +pub struct MemoryHelper { + memory: Memory, +} + +impl MemoryHelper { + pub fn new(memory: Memory) -> Self { + Self { memory } + } + + /// 从 caller 中拿到 `memory` export 的 helper（在每个 host fn 起始处调用）。 + pub fn from_caller(caller: &mut Caller<'_, T>) -> Result { + let Some(mem) = caller.get_export("memory").and_then(|e| e.into_memory()) else { + return Err(WasmHostError::AbiViolation( + "guest module has no `memory` export".to_string(), + )); + }; + Ok(Self { memory: mem }) + } + + /// 读取 guest 线性内存 `[ptr, ptr+len)` 的字节切片。 + pub fn read_bytes(&self, store: StoreContext<'_, T>, ptr: u32, len: u32) -> Result, WasmHostError> { + let data = self.memory.data(&store); + let start = ptr as usize; + let end = start.saturating_add(len as usize); + if end > data.len() { + return Err(WasmHostError::MemoryOob { ptr, len }); + } + Ok(data[start..end].to_vec()) + } + + /// 读 UTF-8 字符串；非法 UTF-8 用 lossy 转换，不报错。 + pub fn read_string_lossy(&self, store: StoreContext<'_, T>, ptr: u32, len: u32) -> Result { + let bytes = self.read_bytes(store, ptr, len)?; + Ok(String::from_utf8_lossy(&bytes).into_owned()) + } + + /// 把 host 数据写入 guest 已经分配好的 `ptr` 处。 + pub fn write_bytes(&self, mut store: StoreContextMut<'_, T>, ptr: u32, data: &[u8]) -> Result<(), WasmHostError> { + let mem = self.memory.data_mut(&mut store); + let start = ptr as usize; + let end = start.saturating_add(data.len()); + if end > mem.len() { + return Err(WasmHostError::MemoryOob { + ptr, + len: data.len() as u32, + }); + } + mem[start..end].copy_from_slice(data); + Ok(()) + } + + /// 写入一个 little-endian i32 到 guest 内存。 + pub fn write_u32(&self, store: StoreContextMut<'_, T>, ptr: u32, value: u32) -> Result<(), WasmHostError> { + self.write_bytes(store, ptr, &value.to_le_bytes()) + } + + /// 写入一个 little-endian u64 到 guest 内存。 + pub fn write_u64(&self, store: StoreContextMut<'_, T>, ptr: u32, value: u64) -> Result<(), WasmHostError> { + self.write_bytes(store, ptr, &value.to_le_bytes()) + } +} + +// ───────────────────────────────────────────────────────── +// header / call pairs 的二进制布局编解码 +// ───────────────────────────────────────────────────────── +// +// proxy-wasm header pairs 序列化结构（little-endian）： +// ``` +// u32 count +// repeat count: u32 key_size, u32 value_size +// repeat count: key_bytes, \0, value_bytes, \0 +// ``` +// `\0` 是为 C 互操作而保留的尾字节；rust 解码端会忽略它。 +// 编码侧也按规范追加 `\0`。 + +pub fn encode_pairs(pairs: &[(&[u8], &[u8])]) -> Vec { + let count = pairs.len() as u32; + // 估算容量：头 4 + 每对 8 + 每对 (k+1+v+1) + let mut cap: usize = 4 + pairs.len() * 8; + for (k, v) in pairs { + cap += k.len() + 1 + v.len() + 1; + } + let mut out = Vec::with_capacity(cap); + out.extend_from_slice(&count.to_le_bytes()); + for (k, v) in pairs { + out.extend_from_slice(&(k.len() as u32).to_le_bytes()); + out.extend_from_slice(&(v.len() as u32).to_le_bytes()); + } + for (k, v) in pairs { + out.extend_from_slice(k); + out.push(0); + out.extend_from_slice(v); + out.push(0); + } + out +} + +/// 解码 `proxy_set_header_map_pairs` 写入的字节流为 (key, value) 列表。 +/// +/// 严格按编码格式校验长度；不合法直接返回 `None`，由 host 端转 BadArgument。 +pub fn decode_pairs(bytes: &[u8]) -> Option, Vec)>> { + if bytes.len() < 4 { + return None; + } + let mut pos = 0; + let count = u32_from_slice(bytes, pos)? as usize; + pos += 4; + if bytes.len() < 4 + count * 8 { + return None; + } + let mut sizes = Vec::with_capacity(count); + for _ in 0..count { + let k = u32_from_slice(bytes, pos)? as usize; + pos += 4; + let v = u32_from_slice(bytes, pos)? as usize; + pos += 4; + sizes.push((k, v)); + } + let mut out = Vec::with_capacity(count); + for (ks, vs) in sizes { + if pos + ks + 1 + vs + 1 > bytes.len() { + return None; + } + let key = bytes[pos..pos + ks].to_vec(); + pos += ks + 1; // skip \0 + let val = bytes[pos..pos + vs].to_vec(); + pos += vs + 1; + out.push((key, val)); + } + Some(out) +} + +#[inline] +fn u32_from_slice(bytes: &[u8], pos: usize) -> Option { + let s = bytes.get(pos..pos + 4)?; + let arr: [u8; 4] = s.try_into().ok()?; + Some(u32::from_le_bytes(arr)) +} diff --git a/crates/plugin-wasm/src/config.rs b/crates/plugin-wasm/src/config.rs new file mode 100644 index 00000000..af433de2 --- /dev/null +++ b/crates/plugin-wasm/src/config.rs @@ -0,0 +1,96 @@ +//! `WasmPluginShell` 的 JSON spec（与演进文档 §5 对齐）。 + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum FailStrategy { + #[default] + FailOpen, + FailClose, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct WasmLimits { + #[serde(default)] + pub max_memory_pages: Option, + #[serde(default)] + pub fuel_per_call: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct WasmPluginShellConfig { + /// `file://`、`http(s)://` 或本地路径。 + pub url: String, + /// 传给 guest `proxy_on_configure` 的配置：可为 JSON 对象；序列化为 YAML 字节给 hai 系插件。 + #[serde(default)] + pub plugin_config: serde_json::Value, + #[serde(default)] + pub fail_strategy: FailStrategy, + /// `dispatch_http_call` 时 guest 传入的 cluster 名 → 真实 HTTP base URL。 + /// + /// 兼容 hai 的 Higress cluster 写法 `outbound|||.`： + /// 若直接命中则用配置 base，否则 host 会回退用 `:authority` header（hai 已带）发起请求。 + #[serde(default)] + pub clusters: HashMap, + #[serde(default)] + pub limits: WasmLimits, + /// 创建时是否尝试用占位 linker 实例化一次（尽早发现链接错误）。当前实现已弃用，保留兼容字段。 + #[serde(default = "default_validate")] + pub validate_on_create: bool, +} + +fn default_validate() -> bool { + false +} + +impl Default for WasmPluginShellConfig { + fn default() -> Self { + Self { + url: String::new(), + plugin_config: serde_json::Value::Null, + fail_strategy: FailStrategy::FailOpen, + clusters: HashMap::new(), + limits: WasmLimits::default(), + validate_on_create: false, + } + } +} + +impl WasmPluginShellConfig { + /// 把 `plugin_config`（任意 JSON）转换为 hai 风格 YAML 字节流。 + /// + /// hai-process-mix 在 `on_configure` 内是 `serde_yaml::from_slice::(&bytes)`， + /// 所以无论上层用 JSON 还是 YAML 写，传给 guest 的都必须是 YAML 序列化结果。 + pub fn configuration_bytes(&self) -> Vec { + if self.plugin_config.is_null() { + return Vec::new(); + } + serde_yaml::to_string(&self.plugin_config) + .unwrap_or_default() + .into_bytes() + } + + /// 给定 guest 传来的 cluster 字符串，返回基础 URL（`http://host:port`）。 + /// + /// 优先精确匹配配置 map；其次尝试解析 Envoy/Higress 习惯写法 + /// `outbound|||` -> `http://:`； + /// 都不命中返回 `None`。 + pub fn resolve_cluster(&self, cluster: &str) -> Option { + if let Some(v) = self.clusters.get(cluster) { + return Some(v.clone()); + } + if let Some(rest) = cluster.strip_prefix("outbound|") { + let mut parts = rest.splitn(2, "||"); + let port = parts.next()?.trim(); + let host = parts.next()?.trim(); + if host.is_empty() || port.is_empty() { + return None; + } + return Some(format!("http://{host}:{port}")); + } + None + } +} diff --git a/crates/plugin-wasm/src/engine.rs b/crates/plugin-wasm/src/engine.rs new file mode 100644 index 00000000..e62d2060 --- /dev/null +++ b/crates/plugin-wasm/src/engine.rs @@ -0,0 +1,24 @@ +//! 共享 `wasmtime::Engine`：同进程内所有 wasm 插件实例共用。 +//! +//! **同步模式**：host fn 是 sync，故不能开 `async_support`——否则 host fn 内 +//! 调 guest 的 `proxy_on_memory_allocate` 会 panic「must use `call_async` with async stores」。 +//! `proxy_http_call` 的异步语义通过 `tokio::spawn` + mpsc channel 实现， +//! 不需要把整个 store 切到 async。 +//! +//! 资源/超时限制（fuel/epoch）暂未启用：演进文档 §4.7 的"资源/Panic 隔离" +//! 列入后续阶段；本阶段优先保证 hai-process-mix 鉴权流程跑通。 + +use once_cell::sync::OnceCell; +use wasmtime::{Config, Engine}; + +static ENGINE: OnceCell = OnceCell::new(); + +/// 进程级单例 Engine（multi-memory 开，async 关）。 +pub fn shared_engine() -> &'static Engine { + ENGINE.get_or_init(|| { + let mut cfg = Config::new(); + cfg.wasm_multi_memory(true); + cfg.async_support(false); + Engine::new(&cfg).expect("wasmtime Engine::new") + }) +} diff --git a/crates/plugin-wasm/src/error.rs b/crates/plugin-wasm/src/error.rs new file mode 100644 index 00000000..bf584687 --- /dev/null +++ b/crates/plugin-wasm/src/error.rs @@ -0,0 +1,21 @@ +//! WASM 插件宿主侧错误类型（实现 `std::error::Error`，可自动装箱为 `BoxError`）。 + +#[derive(Debug, thiserror::Error)] +pub enum WasmHostError { + #[error("fetch wasm: {0}")] + Fetch(String), + #[error("wasmtime: {0}")] + Wasmtime(#[from] wasmtime::Error), + #[error("instantiation failed: {0}")] + Instantiate(String), + #[error("guest abi violation: {0}")] + AbiViolation(String), + #[error("memory oob: ptr={ptr} len={len}")] + MemoryOob { ptr: u32, len: u32 }, + #[error("wasm guest trap during {hook}: {source}")] + GuestTrap { hook: &'static str, source: wasmtime::Error }, + #[error("dispatch_http_call: {0}")] + Dispatch(String), + #[error("config: {0}")] + Config(String), +} diff --git a/crates/plugin-wasm/src/fetch.rs b/crates/plugin-wasm/src/fetch.rs new file mode 100644 index 00000000..8fa3a37f --- /dev/null +++ b/crates/plugin-wasm/src/fetch.rs @@ -0,0 +1,19 @@ +//! 同步拉取 WASM 字节（在 `Plugin::create` 同步上下文中使用）。 +//! +//! 支持：`file://...` 与裸文件系统路径；`http(s)://...` 暂未在 reqwest blocking 下启用， +//! 后续按 OCI 接入时一起做。 + +use crate::error::WasmHostError; + +pub fn fetch_wasm_bytes_sync(url_or_path: &str) -> Result, WasmHostError> { + let trim = url_or_path.trim(); + if let Some(rest) = trim.strip_prefix("file://") { + return std::fs::read(rest).map_err(|e| WasmHostError::Fetch(format!("read file {rest}: {e}"))); + } + if trim.starts_with("http://") || trim.starts_with("https://") { + return Err(WasmHostError::Fetch( + "http(s)://wasm 拉取暂未启用：请使用 file:// 或裸路径".to_string(), + )); + } + std::fs::read(trim).map_err(|e| WasmHostError::Fetch(format!("read path {trim}: {e}"))) +} diff --git a/crates/plugin-wasm/src/host_fn.rs b/crates/plugin-wasm/src/host_fn.rs new file mode 100644 index 00000000..3b0f764b --- /dev/null +++ b/crates/plugin-wasm/src/host_fn.rs @@ -0,0 +1,970 @@ +//! 把 proxy-wasm 0.2.x 的全部 host fn 注册到 `wasmtime::Linker`。 +//! +//! 实现策略： +//! +//! - 全部使用 **同步** `func_wrap`（host 端不需要 await）。 +//! - `proxy_http_call` 是唯一的"异步"——它**同步**返回 token，把真正的 HTTP 调用 `tokio::spawn` +//! 出去，结果通过 `dispatch_tx` 投递回 Vm 状态机；Vm 主循环 await。 +//! - hai-process-mix 没用到的能力（grpc_*、shared_data、foreign_function、queue） +//! 全部 stub 为 `Status::Unimplemented`（i32 = 12）以避免 wasmtime instantiate 失败。 +//! +//! 命名与 proxy-wasm spec 完全一致；参数按 i32（线性内存偏移/长度均为 i32）。 + +use std::time::Duration; + +use bytes::Bytes; +use http::{HeaderMap, HeaderName, HeaderValue}; +use tracing::{debug, info, warn}; +use wasmtime::{AsContext, AsContextMut, Caller, Linker}; + +use crate::abi::{decode_pairs, encode_pairs, log_level_to_tracing, BufferType, MapType, MemoryHelper, Status, StreamType}; +use crate::host_state::{HostState, HttpCallResult, LocalResponse}; + +/// 把所有 hai-process-mix 用到的 host fn 注册到 linker。 +/// +/// `dispatch_tx` 用于把异步 HTTP 调用结果发送给 Vm 状态机。 +pub fn register_all( + linker: &mut Linker, + dispatch_tx: tokio::sync::mpsc::UnboundedSender<(u32, HttpCallResult)>, +) -> Result<(), wasmtime::Error> { + // ─────────── proxy_log ─────────── + linker.func_wrap( + "env", + "proxy_log", + |mut caller: Caller<'_, HostState>, level: i32, msg_ptr: i32, msg_size: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let msg = mem + .read_string_lossy(caller.as_context(), msg_ptr as u32, msg_size as u32) + .unwrap_or_default(); + let lvl = log_level_to_tracing(level); + match lvl { + tracing::Level::TRACE => tracing::trace!(target: "spacegate_plugin_wasm::guest", "{msg}"), + tracing::Level::DEBUG => tracing::debug!(target: "spacegate_plugin_wasm::guest", "{msg}"), + tracing::Level::INFO => tracing::info!(target: "spacegate_plugin_wasm::guest", "{msg}"), + tracing::Level::WARN => tracing::warn!(target: "spacegate_plugin_wasm::guest", "{msg}"), + tracing::Level::ERROR => tracing::error!(target: "spacegate_plugin_wasm::guest", "{msg}"), + } + Status::Ok.as_i32() + }, + )?; + + // ─────────── proxy_get_current_time_nanoseconds(return_time_ptr) ─────────── + linker.func_wrap( + "env", + "proxy_get_current_time_nanoseconds", + |mut caller: Caller<'_, HostState>, return_ptr: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0); + if mem.write_u64(caller.as_context_mut(), return_ptr as u32, nanos).is_err() { + return Status::InternalFailure.as_i32(); + } + Status::Ok.as_i32() + }, + )?; + + // ─────────── proxy_set_tick_period_milliseconds(period) ─────────── + linker.func_wrap( + "env", + "proxy_set_tick_period_milliseconds", + |mut caller: Caller<'_, HostState>, period: i32| -> i32 { + caller.data_mut().tick_period_ms = if period > 0 { Some(period as u32) } else { None }; + Status::Ok.as_i32() + }, + )?; + + // ─────────── proxy_set_effective_context(context_id) ─────────── + linker.func_wrap( + "env", + "proxy_set_effective_context", + |mut caller: Caller<'_, HostState>, ctx_id: i32| -> i32 { + caller.data_mut().effective_context = ctx_id as u32; + Status::Ok.as_i32() + }, + )?; + + // ─────────── proxy_done ─────────── + linker.func_wrap("env", "proxy_done", |_caller: Caller<'_, HostState>| -> i32 { Status::Ok.as_i32() })?; + + // ─────────── proxy_continue_stream(stream_type) ─────────── + // + // hai 通过 `resume_http_request()` 调它，stream_type=0 表示 Request。 + // host 端把当前 ctx 的 continue_requested 置 true，Vm 状态机据此退出 await loop。 + linker.func_wrap( + "env", + "proxy_continue_stream", + |mut caller: Caller<'_, HostState>, stream_type: i32| -> i32 { + let st = caller.data(); + let ctx_id = st.effective_context; + let _ = StreamType::from_i32(stream_type); + if let Some(ctx) = caller.data_mut().contexts.get_mut(&ctx_id) { + ctx.continue_requested = true; + } + Status::Ok.as_i32() + }, + )?; + + // ─────────── proxy_close_stream(stream_type) ─────────── + linker.func_wrap( + "env", + "proxy_close_stream", + |_caller: Caller<'_, HostState>, _stream_type: i32| -> i32 { Status::Ok.as_i32() }, + )?; + + // ─────────── proxy_get_buffer_bytes ─────────── + // + // 签名：(buffer_type, start, max_size, return_data_ptr, return_size_ptr) -> Status + // host 端要： + // 1. 从 HostState 拿对应 buffer（plugin_config / request_body / response_body / call_response_body） + // 2. 调 guest 的 `proxy_on_memory_allocate` 让它给一块缓冲 + // 3. 把字节写到 guest 内存，写回 *return_data = ptr, *return_size = len + linker.func_wrap( + "env", + "proxy_get_buffer_bytes", + |mut caller: Caller<'_, HostState>, + buffer_type: i32, + start: i32, + max_size: i32, + return_data_ptr: i32, + return_size_ptr: i32| + -> i32 { + let buf_type = BufferType::from_i32(buffer_type); + let bytes_opt: Option> = match buf_type { + BufferType::PluginConfiguration | BufferType::VmConfiguration => Some(caller.data().configuration.clone()), + BufferType::HttpRequestBody => caller + .data() + .current_context() + .and_then(|c| c.request_body.as_ref().map(|b| b.to_vec())), + BufferType::HttpResponseBody => caller + .data() + .current_context() + .and_then(|c| c.response_body.as_ref().map(|b| b.to_vec())), + BufferType::HttpCallResponseBody => caller + .data() + .current_context() + .map(|c| c.last_call_body.to_vec()), + BufferType::Unknown(_) => None, + }; + let bytes = match bytes_opt { + Some(b) => b, + None => return Status::NotFound.as_i32(), + }; + // 截取 [start, start + max_size)；max_size 是 u32 reinterpret 进来的， + // proxy-wasm-rust-sdk 经常传 usize::MAX -> u32::MAX，所以这里按 u32 重新解释。 + let start = (start as u32) as usize; + let max_size = (max_size as u32) as usize; + if start > bytes.len() { + return Status::BadArgument.as_i32(); + } + let end = (start.saturating_add(max_size)).min(bytes.len()); + let slice = &bytes[start..end]; + // 空 buffer：写回 (0, 0) 并返回 Ok，让 guest 知道存在但是 0 长度 + if slice.is_empty() { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let _ = mem.write_u32(caller.as_context_mut(), return_data_ptr as u32, 0); + let _ = mem.write_u32(caller.as_context_mut(), return_size_ptr as u32, 0); + return Status::Ok.as_i32(); + } + // 让 guest 分配 + let alloc = match caller.data().alloc.clone() { + Some(f) => f, + None => return Status::InternalFailure.as_i32(), + }; + let guest_ptr = match alloc.call(&mut caller, slice.len() as u32) { + Ok(p) => p, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + if mem.write_bytes(caller.as_context_mut(), guest_ptr, slice).is_err() { + return Status::InternalFailure.as_i32(); + } + let _ = mem.write_u32(caller.as_context_mut(), return_data_ptr as u32, guest_ptr); + let _ = mem.write_u32(caller.as_context_mut(), return_size_ptr as u32, slice.len() as u32); + Status::Ok.as_i32() + }, + )?; + + // ─────────── proxy_set_buffer_bytes ─────────── + // + // 用于 guest 写回 response body（流式 hai 才会用，本阶段不实现完整流式，但 spec 要求接口存在）。 + linker.func_wrap( + "env", + "proxy_set_buffer_bytes", + |mut caller: Caller<'_, HostState>, + buffer_type: i32, + _start: i32, + _size: i32, + data_ptr: i32, + data_size: i32| + -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let bytes = match mem.read_bytes(caller.as_context(), data_ptr as u32, data_size as u32) { + Ok(b) => b, + Err(_) => return Status::BadArgument.as_i32(), + }; + let bt = BufferType::from_i32(buffer_type); + let ctx_id = caller.data().effective_context; + if let Some(ctx) = caller.data_mut().contexts.get_mut(&ctx_id) { + match bt { + BufferType::HttpRequestBody => ctx.request_body = Some(Bytes::from(bytes)), + BufferType::HttpResponseBody => ctx.response_body = Some(Bytes::from(bytes)), + _ => return Status::BadArgument.as_i32(), + } + } + Status::Ok.as_i32() + }, + )?; + + // ─────────── proxy_get_header_map_value ─────────── + linker.func_wrap( + "env", + "proxy_get_header_map_value", + |mut caller: Caller<'_, HostState>, + map_type: i32, + key_ptr: i32, + key_size: i32, + return_data_ptr: i32, + return_size_ptr: i32| + -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let key = match mem.read_string_lossy(caller.as_context(), key_ptr as u32, key_size as u32) { + Ok(s) => s, + Err(_) => return Status::BadArgument.as_i32(), + }; + let key_l = key.to_ascii_lowercase(); + let mt = MapType::from_i32(map_type); + let value_opt = lookup_header(caller.data(), mt, &key_l); + let Some(value) = value_opt else { + return Status::NotFound.as_i32(); + }; + let bytes = value.into_bytes(); + // 空字符串也要分配 0 长度 + let alloc = match caller.data().alloc.clone() { + Some(f) => f, + None => return Status::InternalFailure.as_i32(), + }; + let guest_ptr = if bytes.is_empty() { + 0 + } else { + match alloc.call(&mut caller, bytes.len() as u32) { + Ok(p) => p, + Err(_) => return Status::InternalFailure.as_i32(), + } + }; + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + if guest_ptr > 0 { + if mem.write_bytes(caller.as_context_mut(), guest_ptr, &bytes).is_err() { + return Status::InternalFailure.as_i32(); + } + } + let _ = mem.write_u32(caller.as_context_mut(), return_data_ptr as u32, guest_ptr); + let _ = mem.write_u32(caller.as_context_mut(), return_size_ptr as u32, bytes.len() as u32); + Status::Ok.as_i32() + }, + )?; + + // ─────────── proxy_add_header_map_value ─────────── + linker.func_wrap( + "env", + "proxy_add_header_map_value", + |mut caller: Caller<'_, HostState>, + map_type: i32, + key_ptr: i32, + key_size: i32, + value_ptr: i32, + value_size: i32| + -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let key = match mem.read_string_lossy(caller.as_context(), key_ptr as u32, key_size as u32) { + Ok(s) => s, + Err(_) => return Status::BadArgument.as_i32(), + }; + let value = match mem.read_string_lossy(caller.as_context(), value_ptr as u32, value_size as u32) { + Ok(s) => s, + Err(_) => return Status::BadArgument.as_i32(), + }; + mutate_header(caller.data_mut(), MapType::from_i32(map_type), &key, HeaderMutation::Add(value)) + }, + )?; + + // ─────────── proxy_replace_header_map_value ─────────── + linker.func_wrap( + "env", + "proxy_replace_header_map_value", + |mut caller: Caller<'_, HostState>, + map_type: i32, + key_ptr: i32, + key_size: i32, + value_ptr: i32, + value_size: i32| + -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let key = match mem.read_string_lossy(caller.as_context(), key_ptr as u32, key_size as u32) { + Ok(s) => s, + Err(_) => return Status::BadArgument.as_i32(), + }; + let value = match mem.read_string_lossy(caller.as_context(), value_ptr as u32, value_size as u32) { + Ok(s) => s, + Err(_) => return Status::BadArgument.as_i32(), + }; + mutate_header(caller.data_mut(), MapType::from_i32(map_type), &key, HeaderMutation::Replace(value)) + }, + )?; + + // ─────────── proxy_remove_header_map_value ─────────── + linker.func_wrap( + "env", + "proxy_remove_header_map_value", + |mut caller: Caller<'_, HostState>, map_type: i32, key_ptr: i32, key_size: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let key = match mem.read_string_lossy(caller.as_context(), key_ptr as u32, key_size as u32) { + Ok(s) => s, + Err(_) => return Status::BadArgument.as_i32(), + }; + mutate_header(caller.data_mut(), MapType::from_i32(map_type), &key, HeaderMutation::Remove) + }, + )?; + + // ─────────── proxy_get_header_map_pairs ─────────── + linker.func_wrap( + "env", + "proxy_get_header_map_pairs", + |mut caller: Caller<'_, HostState>, map_type: i32, return_data_ptr: i32, return_size_ptr: i32| -> i32 { + let mt = MapType::from_i32(map_type); + let pairs = collect_pairs(caller.data(), mt); + let buf = { + let refs: Vec<(&[u8], &[u8])> = pairs.iter().map(|(k, v)| (k.as_slice(), v.as_slice())).collect(); + encode_pairs(&refs) + }; + let alloc = match caller.data().alloc.clone() { + Some(f) => f, + None => return Status::InternalFailure.as_i32(), + }; + let guest_ptr = match alloc.call(&mut caller, buf.len() as u32) { + Ok(p) => p, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + if mem.write_bytes(caller.as_context_mut(), guest_ptr, &buf).is_err() { + return Status::InternalFailure.as_i32(); + } + let _ = mem.write_u32(caller.as_context_mut(), return_data_ptr as u32, guest_ptr); + let _ = mem.write_u32(caller.as_context_mut(), return_size_ptr as u32, buf.len() as u32); + Status::Ok.as_i32() + }, + )?; + + // ─────────── proxy_set_header_map_pairs ─────────── + linker.func_wrap( + "env", + "proxy_set_header_map_pairs", + |mut caller: Caller<'_, HostState>, map_type: i32, data_ptr: i32, data_size: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let raw = match mem.read_bytes(caller.as_context(), data_ptr as u32, data_size as u32) { + Ok(b) => b, + Err(_) => return Status::BadArgument.as_i32(), + }; + let Some(pairs) = decode_pairs(&raw) else { + return Status::BadArgument.as_i32(); + }; + let mt = MapType::from_i32(map_type); + let new_map = pairs_to_header_map(&pairs); + replace_map(caller.data_mut(), mt, new_map); + Status::Ok.as_i32() + }, + )?; + + // ─────────── proxy_get_property ─────────── + // + // hai 用它读 `source.address`（客户端 IP）。我们把请求里能拿到的 source ip 提前 + // 放到 ctx.request_pseudo 或 properties 表里，host fn 这里检索。 + linker.func_wrap( + "env", + "proxy_get_property", + |mut caller: Caller<'_, HostState>, path_ptr: i32, path_size: i32, return_data_ptr: i32, return_size_ptr: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let raw = match mem.read_bytes(caller.as_context(), path_ptr as u32, path_size as u32) { + Ok(b) => b, + Err(_) => return Status::BadArgument.as_i32(), + }; + // path 是用 '\0' 分割的多段（proxy-wasm 约定） + let segments: Vec<&[u8]> = raw.split(|b| *b == 0u8).filter(|s| !s.is_empty()).collect(); + // 我们暂时仅识别 `source.address` 一种（hai 唯一用例）。 + let value: Option> = if segments == [b"source".as_slice(), b"address".as_slice()] { + // 优先从 :authority / x-forwarded-for 推导（无客户端 socket 信息时降级） + caller + .data() + .current_context() + .and_then(|c| { + if !c.request_pseudo.authority.is_empty() { + Some(c.request_pseudo.authority.clone()) + } else { + None + } + }) + .map(|s| s.into_bytes()) + } else { + None + }; + let Some(bytes) = value else { + return Status::NotFound.as_i32(); + }; + let alloc = match caller.data().alloc.clone() { + Some(f) => f, + None => return Status::InternalFailure.as_i32(), + }; + let guest_ptr = match alloc.call(&mut caller, bytes.len() as u32) { + Ok(p) => p, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + if mem.write_bytes(caller.as_context_mut(), guest_ptr, &bytes).is_err() { + return Status::InternalFailure.as_i32(); + } + let _ = mem.write_u32(caller.as_context_mut(), return_data_ptr as u32, guest_ptr); + let _ = mem.write_u32(caller.as_context_mut(), return_size_ptr as u32, bytes.len() as u32); + Status::Ok.as_i32() + }, + )?; + + // ─────────── proxy_set_property ───────────（stub） + linker.func_wrap( + "env", + "proxy_set_property", + |_caller: Caller<'_, HostState>, _p_ptr: i32, _p_size: i32, _v_ptr: i32, _v_size: i32| -> i32 { + Status::Ok.as_i32() + }, + )?; + + // ─────────── proxy_send_local_response ─────────── + // + // 签名：(status, status_text_data, status_text_size, body_data, body_size, + // additional_headers_data, additional_headers_size, grpc_status) -> Status + linker.func_wrap( + "env", + "proxy_send_local_response", + |mut caller: Caller<'_, HostState>, + status: i32, + _status_text_data: i32, + _status_text_size: i32, + body_data: i32, + body_size: i32, + headers_data: i32, + headers_size: i32, + _grpc_status: i32| + -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let body = if body_size > 0 { + mem.read_bytes(caller.as_context(), body_data as u32, body_size as u32) + .unwrap_or_default() + } else { + Vec::new() + }; + let headers_bytes = if headers_size > 0 { + mem.read_bytes(caller.as_context(), headers_data as u32, headers_size as u32) + .unwrap_or_default() + } else { + Vec::new() + }; + let pairs = decode_pairs(&headers_bytes).unwrap_or_default(); + let map = pairs_to_header_map(&pairs); + let ctx_id = caller.data().effective_context; + if let Some(ctx) = caller.data_mut().contexts.get_mut(&ctx_id) { + ctx.local_response = Some(LocalResponse { + status: status as u16, + headers: map, + body: Bytes::from(body), + }); + debug!(target: "spacegate_plugin_wasm", ctx_id, status, "guest send_local_response captured"); + } else { + warn!(target: "spacegate_plugin_wasm", ctx_id, "send_local_response on unknown ctx"); + } + Status::Ok.as_i32() + }, + )?; + + // ─────────── proxy_http_call ─────────── + // + // 签名（type 15）： + // (upstream_data, upstream_size, headers_data, headers_size, + // body_data, body_size, trailers_data, trailers_size, timeout_ms, return_token_ptr) -> Status + linker.func_wrap( + "env", + "proxy_http_call", + { + let dispatch_tx = dispatch_tx.clone(); + move |mut caller: Caller<'_, HostState>, + upstream_data: i32, + upstream_size: i32, + headers_data: i32, + headers_size: i32, + body_data: i32, + body_size: i32, + _trailers_data: i32, + _trailers_size: i32, + timeout_ms: i32, + return_token_ptr: i32| + -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let cluster = match mem.read_string_lossy(caller.as_context(), upstream_data as u32, upstream_size as u32) { + Ok(s) => s, + Err(_) => return Status::BadArgument.as_i32(), + }; + let headers_bytes = mem.read_bytes(caller.as_context(), headers_data as u32, headers_size as u32).unwrap_or_default(); + let body = if body_size > 0 { + mem.read_bytes(caller.as_context(), body_data as u32, body_size as u32).unwrap_or_default() + } else { + Vec::new() + }; + let pairs = decode_pairs(&headers_bytes).unwrap_or_default(); + // 解析 :method / :path / :authority + let mut method = "GET".to_string(); + let mut path = "/".to_string(); + let mut authority = String::new(); + let mut others = Vec::with_capacity(pairs.len()); + for (k, v) in &pairs { + let key_str = String::from_utf8_lossy(k); + let val_str = String::from_utf8_lossy(v).into_owned(); + match key_str.as_ref() { + ":method" => method = val_str, + ":path" => path = val_str, + ":authority" => authority = val_str, + ":scheme" => { /* host 层只用 http */ } + _ => others.push((key_str.to_string(), val_str)), + } + } + // cluster → base URL + let st = caller.data(); + let base = st.shell_cfg.resolve_cluster(&cluster).or_else(|| { + if !authority.is_empty() { + Some(format!("http://{authority}")) + } else { + None + } + }); + let Some(base) = base else { + warn!(target: "spacegate_plugin_wasm", cluster = %cluster, "dispatch_http_call: cluster not configured"); + return Status::BadArgument.as_i32(); + }; + let url = format!("{}{}", base.trim_end_matches('/'), path); + let token = caller.data_mut().next_dispatch_token(); + let source_ctx = caller.data().effective_context; + caller.data_mut().pending_calls.insert( + token, + crate::host_state::PendingCall { + waker: None, + source_context_id: source_ctx, + }, + ); + let client = caller.data().http_client.clone(); + let timeout = Duration::from_millis(timeout_ms.max(1) as u64); + let tx = dispatch_tx.clone(); + tokio::spawn(async move { + debug!(target: "spacegate_plugin_wasm", %url, %method, "dispatch_http_call begin"); + let parsed_method = match method.parse::() { + Ok(m) => m, + Err(_) => reqwest::Method::GET, + }; + let mut req = client.request(parsed_method, &url); + for (k, v) in others { + // 跳过 hop-by-hop / 非法字符头 + if k.starts_with(':') { + continue; + } + if let (Ok(name), Ok(val)) = (HeaderName::try_from(k.as_str()), HeaderValue::try_from(v.as_str())) { + req = req.header(name, val); + } + } + if !body.is_empty() { + req = req.body(body); + } + req = req.timeout(timeout); + let result = match req.send().await { + Ok(resp) => { + let status = resp.status().as_u16(); + let mut hdrs = HeaderMap::new(); + for (k, v) in resp.headers().iter() { + if let (Ok(name), Ok(val)) = + (HeaderName::try_from(k.as_str()), HeaderValue::from_bytes(v.as_bytes())) + { + hdrs.append(name, val); + } + } + let body_bytes = resp.bytes().await.unwrap_or_default(); + HttpCallResult { + status, + headers: hdrs, + body: body_bytes, + } + } + Err(e) => { + warn!(target: "spacegate_plugin_wasm", %url, error = %e, "dispatch_http_call failed"); + HttpCallResult { + status: 0, + headers: HeaderMap::new(), + body: Bytes::new(), + } + } + }; + debug!(target: "spacegate_plugin_wasm", token, status = result.status, "dispatch_http_call done"); + let _ = tx.send((token, result)); + }); + // 写回 token + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InternalFailure.as_i32(), + }; + let _ = mem.write_u32(caller.as_context_mut(), return_token_ptr as u32, token); + info!(target: "spacegate_plugin_wasm", token, cluster = %cluster, "dispatch_http_call enqueued"); + Status::Ok.as_i32() + } + }, + )?; + + // ─────────── 其余 hai-process-mix 模块声明但不用的 host fn，全部 stub 为 Ok ─────────── + // + // wasmtime 要求 instantiate 时所有 import 都已 link；这里返回 Ok 不影响功能。 + stub_all_unused(linker)?; + + Ok(()) +} + +// ───────────────────────────────────────────────────────── +// 辅助：lookup / mutate / collect +// ───────────────────────────────────────────────────────── + +fn lookup_header(state: &HostState, mt: MapType, key_lower: &str) -> Option { + let ctx = state.current_context()?; + let map = match mt { + MapType::HttpRequestHeaders | MapType::HttpRequestTrailers => &ctx.request_headers, + MapType::HttpResponseHeaders | MapType::HttpResponseTrailers => &ctx.response_headers, + MapType::HttpCallResponseHeaders | MapType::HttpCallResponseTrailers => &ctx.last_call_headers, + MapType::Unknown(_) => return None, + }; + // 伪头特判：:status / :method / :path / :authority / :scheme + if let Some(value) = pseudo_lookup(ctx, mt, key_lower) { + return Some(value); + } + let name = HeaderName::try_from(key_lower).ok()?; + let val = map.get(&name)?; + val.to_str().ok().map(|s| s.to_string()) +} + +fn pseudo_lookup(ctx: &crate::host_state::RequestContext, mt: MapType, key: &str) -> Option { + match (mt, key) { + (MapType::HttpRequestHeaders, ":method") => Some(ctx.request_pseudo.method.clone()), + (MapType::HttpRequestHeaders, ":path") => Some(ctx.request_pseudo.path.clone()), + (MapType::HttpRequestHeaders, ":authority") => Some(ctx.request_pseudo.authority.clone()), + (MapType::HttpRequestHeaders, ":scheme") => Some(ctx.request_pseudo.scheme.clone()), + (MapType::HttpResponseHeaders, ":status") => ctx.response_status.map(|s| s.to_string()), + (MapType::HttpCallResponseHeaders, ":status") => { + if ctx.last_call_status > 0 { + Some(ctx.last_call_status.to_string()) + } else { + None + } + } + _ => None, + } +} + +enum HeaderMutation { + Add(String), + Replace(String), + Remove, +} + +fn mutate_header(state: &mut HostState, mt: MapType, key: &str, m: HeaderMutation) -> i32 { + let ctx_id = state.effective_context; + let Some(ctx) = state.contexts.get_mut(&ctx_id) else { + return Status::NotFound.as_i32(); + }; + // 伪头处理（hai 不会改 :path 但理论上要支持） + if key.starts_with(':') { + let new_val = match &m { + HeaderMutation::Add(v) | HeaderMutation::Replace(v) => Some(v.clone()), + HeaderMutation::Remove => None, + }; + match (mt, key) { + (MapType::HttpRequestHeaders, ":path") => { + ctx.request_pseudo.path = new_val.unwrap_or_default(); + } + (MapType::HttpRequestHeaders, ":method") => { + ctx.request_pseudo.method = new_val.unwrap_or_default(); + } + (MapType::HttpRequestHeaders, ":authority") => { + ctx.request_pseudo.authority = new_val.unwrap_or_default(); + } + (MapType::HttpRequestHeaders, ":scheme") => { + ctx.request_pseudo.scheme = new_val.unwrap_or_default(); + } + (MapType::HttpResponseHeaders, ":status") => { + if let Some(v) = new_val { + ctx.response_status = v.parse().ok(); + } + } + _ => {} + } + return Status::Ok.as_i32(); + } + let Ok(name) = HeaderName::try_from(key) else { + return Status::BadArgument.as_i32(); + }; + let map = match mt { + MapType::HttpRequestHeaders | MapType::HttpRequestTrailers => &mut ctx.request_headers, + MapType::HttpResponseHeaders | MapType::HttpResponseTrailers => &mut ctx.response_headers, + MapType::HttpCallResponseHeaders | MapType::HttpCallResponseTrailers => &mut ctx.last_call_headers, + MapType::Unknown(_) => return Status::BadArgument.as_i32(), + }; + match m { + HeaderMutation::Add(v) => { + if let Ok(val) = HeaderValue::try_from(v) { + map.append(name, val); + } + } + HeaderMutation::Replace(v) => { + if let Ok(val) = HeaderValue::try_from(v) { + map.insert(name, val); + } + } + HeaderMutation::Remove => { + map.remove(name); + } + } + Status::Ok.as_i32() +} + +fn collect_pairs(state: &HostState, mt: MapType) -> Vec<(Vec, Vec)> { + let Some(ctx) = state.current_context() else { + return Vec::new(); + }; + let map = match mt { + MapType::HttpRequestHeaders | MapType::HttpRequestTrailers => &ctx.request_headers, + MapType::HttpResponseHeaders | MapType::HttpResponseTrailers => &ctx.response_headers, + MapType::HttpCallResponseHeaders | MapType::HttpCallResponseTrailers => &ctx.last_call_headers, + MapType::Unknown(_) => return Vec::new(), + }; + let mut out: Vec<(Vec, Vec)> = Vec::with_capacity(map.len() + 4); + // 加伪头 + match mt { + MapType::HttpRequestHeaders => { + if !ctx.request_pseudo.method.is_empty() { + out.push((b":method".to_vec(), ctx.request_pseudo.method.as_bytes().to_vec())); + } + if !ctx.request_pseudo.path.is_empty() { + out.push((b":path".to_vec(), ctx.request_pseudo.path.as_bytes().to_vec())); + } + if !ctx.request_pseudo.authority.is_empty() { + out.push((b":authority".to_vec(), ctx.request_pseudo.authority.as_bytes().to_vec())); + } + if !ctx.request_pseudo.scheme.is_empty() { + out.push((b":scheme".to_vec(), ctx.request_pseudo.scheme.as_bytes().to_vec())); + } + } + MapType::HttpResponseHeaders => { + if let Some(s) = ctx.response_status { + out.push((b":status".to_vec(), s.to_string().into_bytes())); + } + } + MapType::HttpCallResponseHeaders => { + if ctx.last_call_status > 0 { + out.push((b":status".to_vec(), ctx.last_call_status.to_string().into_bytes())); + } + } + _ => {} + } + for (k, v) in map.iter() { + out.push((k.as_str().as_bytes().to_vec(), v.as_bytes().to_vec())); + } + out +} + +fn pairs_to_header_map(pairs: &[(Vec, Vec)]) -> HeaderMap { + let mut out = HeaderMap::new(); + for (k, v) in pairs { + let Ok(key) = HeaderName::try_from(k.as_slice()) else { + continue; + }; + let Ok(val) = HeaderValue::from_bytes(v.as_slice()) else { + continue; + }; + out.append(key, val); + } + out +} + +fn replace_map(state: &mut HostState, mt: MapType, new_map: HeaderMap) { + let ctx_id = state.effective_context; + let Some(ctx) = state.contexts.get_mut(&ctx_id) else { + return; + }; + match mt { + MapType::HttpRequestHeaders => ctx.request_headers = new_map, + MapType::HttpResponseHeaders => ctx.response_headers = new_map, + _ => {} + } +} + +// ───────────────────────────────────────────────────────── +// stub：hai 模块声明但本阶段不需要语义的 host fn +// ───────────────────────────────────────────────────────── + +fn stub_all_unused(linker: &mut Linker) -> Result<(), wasmtime::Error> { + // proxy_get_shared_data / proxy_set_shared_data：暂返回 NotFound / Ok + linker.func_wrap( + "env", + "proxy_get_shared_data", + |_caller: Caller<'_, HostState>, _k_ptr: i32, _k_size: i32, _v_ptr: i32, _v_size: i32, _cas_ptr: i32| -> i32 { + Status::NotFound.as_i32() + }, + )?; + linker.func_wrap( + "env", + "proxy_set_shared_data", + |_caller: Caller<'_, HostState>, _k_ptr: i32, _k_size: i32, _v_ptr: i32, _v_size: i32, _cas: i32| -> i32 { + Status::Ok.as_i32() + }, + )?; + + // proxy_get_status：返回 Ok（与本地响应状态码相关，但 hai 不读取） + linker.func_wrap( + "env", + "proxy_get_status", + |_caller: Caller<'_, HostState>, _status_code_ptr: i32, _msg_ptr: i32, _msg_size: i32| -> i32 { Status::Ok.as_i32() }, + )?; + + // 共享队列 + linker.func_wrap( + "env", + "proxy_register_shared_queue", + |_caller: Caller<'_, HostState>, _n_ptr: i32, _n_size: i32, _ret: i32| -> i32 { Status::Empty.as_i32() }, + )?; + linker.func_wrap( + "env", + "proxy_resolve_shared_queue", + |_caller: Caller<'_, HostState>, _vid_ptr: i32, _vid_size: i32, _n_ptr: i32, _n_size: i32, _ret: i32| -> i32 { + Status::Empty.as_i32() + }, + )?; + linker.func_wrap( + "env", + "proxy_enqueue_shared_queue", + |_caller: Caller<'_, HostState>, _qid: i32, _v_ptr: i32, _v_size: i32| -> i32 { Status::Empty.as_i32() }, + )?; + linker.func_wrap( + "env", + "proxy_dequeue_shared_queue", + |_caller: Caller<'_, HostState>, _qid: i32, _v_ptr: i32, _v_size: i32| -> i32 { Status::Empty.as_i32() }, + )?; + + // gRPC 相关全部 Empty + linker.func_wrap( + "env", + "proxy_grpc_call", + |_caller: Caller<'_, HostState>, + _a: i32, + _b: i32, + _c: i32, + _d: i32, + _e: i32, + _f: i32, + _g: i32, + _h: i32, + _i: i32, + _j: i32, + _k: i32, + _l: i32| + -> i32 { Status::Empty.as_i32() }, + )?; + linker.func_wrap( + "env", + "proxy_grpc_stream", + |_caller: Caller<'_, HostState>, + _a: i32, + _b: i32, + _c: i32, + _d: i32, + _e: i32, + _f: i32, + _g: i32, + _h: i32, + _i: i32| + -> i32 { Status::Empty.as_i32() }, + )?; + linker.func_wrap( + "env", + "proxy_grpc_cancel", + |_caller: Caller<'_, HostState>, _t: i32| -> i32 { Status::Empty.as_i32() }, + )?; + linker.func_wrap( + "env", + "proxy_grpc_close", + |_caller: Caller<'_, HostState>, _t: i32| -> i32 { Status::Empty.as_i32() }, + )?; + linker.func_wrap( + "env", + "proxy_grpc_send", + |_caller: Caller<'_, HostState>, _t: i32, _m: i32, _ms: i32, _eos: i32| -> i32 { Status::Empty.as_i32() }, + )?; + + // foreign function：不支持 + linker.func_wrap( + "env", + "proxy_call_foreign_function", + |_caller: Caller<'_, HostState>, _a: i32, _b: i32, _c: i32, _d: i32, _e: i32, _f: i32| -> i32 { + Status::Empty.as_i32() + }, + )?; + + Ok(()) +} diff --git a/crates/plugin-wasm/src/host_state.rs b/crates/plugin-wasm/src/host_state.rs new file mode 100644 index 00000000..f261151b --- /dev/null +++ b/crates/plugin-wasm/src/host_state.rs @@ -0,0 +1,158 @@ +//! 传给 `wasmtime::Store` 的宿主状态。 +//! +//! - 顶层 `HostState` 承载：进程级 reqwest 客户端、shell 配置、序列化后的 plugin_config 字节、 +//! memory / 分配器 export、所有 HTTP 上下文、未完结的 `proxy_http_call` 句柄等。 +//! - 每个 HTTP 请求建一个 [`RequestContext`]，由 `vm.rs` 在调 `proxy_on_*` 钩子前后维护。 +//! - host fn 通过 `caller.data() / data_mut()` 读写 `HostState`，并以 +//! `effective_context` 字段定位「当前是哪个上下文」（hai-process-mix 调 +//! `proxy_set_effective_context` 切换）。 + +use std::collections::HashMap; +use std::sync::Arc; + +use bytes::Bytes; +use http::HeaderMap; +use wasmtime::{Memory, TypedFunc}; + +use crate::config::WasmPluginShellConfig; + +/// 约定的 root context id：proxy-wasm 默认从 1 开始。 +pub const ROOT_CONTEXT_ID: u32 = 1; + +/// HTTP 上下文在生命周期中处于的阶段（vm.rs 调钩子时打标记，host fn 据此判断）。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ContextStage { + #[default] + Init, + RequestHeaders, + RequestBody, + ResponseHeaders, + ResponseBody, + Log, +} + +/// HTTP/2 风格的伪头（`:method` 等）。proxy-wasm guest 通过 header_map 拿到它们， +/// 我们额外用一个结构体专门承载，便于在 `inner.call` 前重建 `Uri`。 +#[derive(Debug, Clone, Default)] +pub struct PseudoHeaders { + pub method: String, + pub path: String, + pub authority: String, + pub scheme: String, +} + +/// guest 调 `proxy_send_local_response` 时 host 捕获的结构。 +#[derive(Debug)] +pub struct LocalResponse { + pub status: u16, + pub headers: HeaderMap, + pub body: Bytes, +} + +/// `proxy_http_call` 的异步结果：spawn 出去的 reqwest 任务通过 channel 把它送回 Vm。 +#[derive(Debug, Default)] +pub struct HttpCallResult { + pub status: u16, + pub headers: HeaderMap, + pub body: Bytes, +} + +/// 一次未完结的 `proxy_http_call`。`source_context_id` 指明该 token 是哪个 ctx 发起的， +/// 这样 Vm 状态机在拿到结果时能恢复到正确的 effective_context 再调 `proxy_on_http_call_response`。 +#[derive(Debug)] +pub struct PendingCall { + #[allow(dead_code)] + pub waker: Option, + pub source_context_id: u32, +} + +/// 单个 HTTP 请求的所有状态（请求/响应头 / body / 上次 dispatch 结果 / 本地响应 / 短路标记）。 +#[derive(Debug, Default)] +pub struct RequestContext { + pub parent_id: u32, + pub stage: ContextStage, + pub request_pseudo: PseudoHeaders, + pub request_headers: HeaderMap, + pub request_body: Option, + pub response_status: Option, + pub response_headers: HeaderMap, + pub response_body: Option, + /// 上次 `proxy_http_call` 回调时由 host 注入；guest 通过 + /// `get_http_call_response_*` 读它。 + pub last_call_headers: HeaderMap, + pub last_call_body: Bytes, + /// 最近一次 dispatch_http_call 返回的状态码（hai 用 `:status` 伪头读取）。 + pub last_call_status: u16, + /// guest 显式 `resume_http_request()` 后置 true；Vm 退出 Pause 等待循环。 + pub continue_requested: bool, + /// guest 调 `send_local_response` 后写入；Vm 据此短路返回。 + pub local_response: Option, +} + +/// 进程内传给 wasmtime `Store` 的状态。生命周期与一次 Vm 实例一致。 +/// +/// 不 derive Debug 因为 `TypedFunc` 不实现 Debug。 +pub struct HostState { + pub shell_cfg: Arc, + /// guest `proxy_on_configure` 读取的字节（来自 shell_cfg.plugin_config 序列化）。 + pub configuration: Vec, + /// guest 导出的线性内存（vm.rs 实例化完成后填）。 + pub memory: Option, + /// guest 导出 `proxy_on_memory_allocate(size) -> ptr`：host 把数据回写到 wasm 前的分配函数。 + pub alloc: Option>, + pub root_context_id: u32, + /// 当前 hostcall 关联的上下文 id（由 vm.rs 在每次钩子前设置， + /// 也可被 guest 的 `proxy_set_effective_context` 覆盖）。 + pub effective_context: u32, + pub contexts: HashMap, + /// guest 调用 `proxy_set_tick_period_milliseconds` 后存这里；Phase 1 暂不真正驱动 tick。 + pub tick_period_ms: Option, + /// 未完结的 dispatch_http_call 句柄表。 + pub pending_calls: HashMap, + /// dispatch token 单调递增计数器。 + next_token: u32, + /// host 端 reqwest 客户端：所有 dispatch_http_call 复用一个，免去握手开销。 + pub http_client: reqwest::Client, +} + +impl HostState { + pub fn new(shell_cfg: Arc) -> Self { + // 把 plugin_config 序列化成 YAML 字节，与 hai-process-mix 的 `serde_yaml::from_slice` 对齐。 + let configuration = shell_cfg.configuration_bytes(); + let http_client = reqwest::Client::builder() + .pool_max_idle_per_host(8) + .build() + .unwrap_or_else(|_| reqwest::Client::new()); + Self { + shell_cfg, + configuration, + memory: None, + alloc: None, + root_context_id: ROOT_CONTEXT_ID, + effective_context: ROOT_CONTEXT_ID, + contexts: HashMap::new(), + tick_period_ms: None, + pending_calls: HashMap::new(), + next_token: 1, + http_client, + } + } + + /// 取当前生效的 ctx 的不可变引用（host fn 大量使用）。 + pub fn current_context(&self) -> Option<&RequestContext> { + self.contexts.get(&self.effective_context) + } + + /// 取当前生效的 ctx 的可变引用。 + #[allow(dead_code)] + pub fn current_context_mut(&mut self) -> Option<&mut RequestContext> { + self.contexts.get_mut(&self.effective_context) + } + + /// 分配下一个 dispatch_http_call token；约定 0 保留，token 从 1 开始单调递增。 + pub fn next_dispatch_token(&mut self) -> u32 { + let t = self.next_token; + self.next_token = self.next_token.wrapping_add(1).max(1); + t + } +} diff --git a/crates/plugin-wasm/src/lib.rs b/crates/plugin-wasm/src/lib.rs new file mode 100644 index 00000000..b0906f01 --- /dev/null +++ b/crates/plugin-wasm/src/lib.rs @@ -0,0 +1,35 @@ +//! SpaceGate **proxy-wasm（wasmtime）** 宿主 crate。 +//! +//! **集成方式**：不要从 `spacegate-plugin` 依赖本 crate（会形成循环依赖），应启用 `spacegate-shell` 的 `plugin-wasm` +//! feature；`spacegate_shell::startup` 会在网关启动时调用 [`register`]。 +//! +//! 当前实现度（对照 `spacegate演进方案-引入proxy-wasm.md` §4）： +//! +//! - ✅ [`WasmPluginShell`]：`Plugin::CODE = "wasm"`，`call` 真正驱动 wasm VM +//! - ✅ [`engine`] / [`runtime`]：进程级 wasmtime Engine + Module 缓存（按 url） +//! - ✅ [`vm::Vm`]：单 VM 异步状态机；驱动 `proxy_on_request_headers` → `proxy_on_http_call_response` → inner.call → `proxy_on_response_headers/body/log/done/delete` +//! - ✅ [`host_fn`]：proxy-wasm ABI 0.2.1 必要子集（log/time/header/buffer/property/local_response/dispatch_http_call/tick/continue/done） +//! - ⏳ 未做：VmPool（每请求新建 Vm）、ScanningBody（流式 SSE 截断）、fuel/epoch 资源隔离 + +#![deny(clippy::unwrap_used, clippy::dbg_macro)] + +pub mod abi; +pub mod config; +pub mod engine; +pub mod error; +pub mod fetch; +pub mod host_fn; +pub mod host_state; +pub mod runtime; +pub mod shell; +pub mod vm; + +pub use config::WasmPluginShellConfig; +pub use shell::WasmPluginShell; + +use spacegate_plugin::PluginRepository; + +/// 向仓库注册 `wasm` 插件类型（需在 `register_prelude` 或启动逻辑中调用一次）。 +pub fn register(repo: &PluginRepository) { + repo.register::(); +} diff --git a/crates/plugin-wasm/src/runtime.rs b/crates/plugin-wasm/src/runtime.rs new file mode 100644 index 00000000..8001d78a --- /dev/null +++ b/crates/plugin-wasm/src/runtime.rs @@ -0,0 +1,45 @@ +//! WASM 模块编译与按 URL 缓存（减少同一 `url` 重复编译）。 + +use std::sync::Arc; + +use moka::sync::Cache; +use once_cell::sync::OnceCell; +use wasmtime::Module; + +use crate::engine::shared_engine; +use crate::error::WasmHostError; +use crate::fetch::fetch_wasm_bytes_sync; + +/// 进程内模块缓存（键：wasm `url` 字符串）。 +pub struct WasmModuleCache { + engine: &'static wasmtime::Engine, + inner: Cache>, +} + +impl WasmModuleCache { + pub fn new(max_entries: u64) -> Self { + Self { + engine: shared_engine(), + inner: Cache::new(max_entries), + } + } + + /// 拉取字节并编译；命中缓存则直接返回 `Arc`。 + pub fn get_or_compile(&self, url: &str) -> Result, WasmHostError> { + let key = url.to_string(); + if let Some(m) = self.inner.get(&key) { + return Ok(m); + } + let bytes = fetch_wasm_bytes_sync(url)?; + let m = Arc::new(Module::new(self.engine, &bytes)?); + self.inner.insert(key, m.clone()); + Ok(m) + } +} + +static CACHE: OnceCell = OnceCell::new(); + +/// 默认缓存（容量 64）；多实例同 URL 共享编译结果。 +pub fn default_module_cache() -> &'static WasmModuleCache { + CACHE.get_or_init(|| WasmModuleCache::new(64)) +} diff --git a/crates/plugin-wasm/src/shell.rs b/crates/plugin-wasm/src/shell.rs new file mode 100644 index 00000000..924db1f4 --- /dev/null +++ b/crates/plugin-wasm/src/shell.rs @@ -0,0 +1,96 @@ +//! `Plugin` 实现：在 `call` 内异步驱动 wasm Vm，按 fail_strategy 处理 Trap。 +//! +//! 首版未做 VM 池：每次请求新建 Vm。hai-process-mix 实例化 + 配置一遍约几毫秒； +//! 后续按演进文档 §4.3 接 `VmPool` 即可显著降损。 + +use std::sync::Arc; + +use spacegate_kernel::{SgBody, SgRequest, SgResponse}; +use spacegate_plugin::{BoxError, Inner, Plugin, PluginConfig}; + +use crate::config::{FailStrategy, WasmPluginShellConfig}; +use crate::runtime::default_module_cache; +use crate::vm::Vm; + +/// Proxy-Wasm 宿主壳插件（`CODE = "wasm"`）。 +pub struct WasmPluginShell { + cfg: Arc, + module: Arc, +} + +impl Plugin for WasmPluginShell { + const CODE: &'static str = "wasm"; + + fn call(&self, req: SgRequest, inner: Inner) -> impl std::future::Future> + Send { + let cfg = self.cfg.clone(); + let module = self.module.clone(); + async move { + tracing::info!( + target: "spacegate_plugin_wasm", + method = %req.method(), + uri = %req.uri(), + "wasm plugin shell: request entered plugin layer" + ); + let vm_res = Vm::new(&module, cfg.clone()).await; + let mut vm = match vm_res { + Ok(v) => v, + Err(e) => { + tracing::error!(target: "spacegate_plugin_wasm", error = %e, "Vm::new failed"); + return Ok(passthrough_on_error(e.to_string(), req, inner, cfg.fail_strategy).await); + } + }; + tracing::info!(target: "spacegate_plugin_wasm", "Vm initialized, entering process"); + match vm.process(req, inner).await { + Ok(resp) => { + tracing::info!(target: "spacegate_plugin_wasm", status = %resp.status(), "Vm::process ok"); + Ok(resp) + } + Err(e) => { + tracing::error!(target: "spacegate_plugin_wasm", error = %e, "wasm plugin failed"); + let status = if matches!(cfg.fail_strategy, FailStrategy::FailOpen) { + http::StatusCode::BAD_GATEWAY + } else { + http::StatusCode::INTERNAL_SERVER_ERROR + }; + let mut resp = SgResponse::new(SgBody::full(format!("wasm plugin error: {e}"))); + *resp.status_mut() = status; + Ok(resp) + } + } + } + } + + fn create(plugin_config: PluginConfig) -> Result { + let raw_spec = plugin_config.spec.clone(); + let cfg: WasmPluginShellConfig = serde_json::from_value(plugin_config.spec).map_err(|e| -> BoxError { format!("wasm spec: {e}").into() })?; + if cfg.url.trim().is_empty() { + return Err("wasm plugin: missing or empty `url`".into()); + } + tracing::info!( + target: "spacegate_plugin_wasm", + url = %cfg.url, + plugin_config_kind = %if cfg.plugin_config.is_null() { "null" } else { "object" }, + plugin_config_keys = ?cfg.plugin_config.as_object().map(|o| o.keys().collect::>()), + clusters = ?cfg.clusters.keys().collect::>(), + raw_keys = ?raw_spec.as_object().map(|o| o.keys().collect::>()), + "wasm plugin: create with config" + ); + let cache = default_module_cache(); + let module = cache.get_or_compile(cfg.url.trim()).map_err(|e| -> BoxError { format!("compile wasm: {e}").into() })?; + Ok(Self { + cfg: Arc::new(cfg), + module, + }) + } +} + +/// 当 Vm::new 失败时，原 `req` 已经被消费；按 fail_strategy 合成一个最简单的兜底响应。 +async fn passthrough_on_error(err: String, _req: SgRequest, _inner: Inner, fs: FailStrategy) -> SgResponse { + let status = match fs { + FailStrategy::FailOpen => http::StatusCode::BAD_GATEWAY, + FailStrategy::FailClose => http::StatusCode::INTERNAL_SERVER_ERROR, + }; + let mut resp = SgResponse::new(SgBody::full(format!("wasm plugin init failed: {err}"))); + *resp.status_mut() = status; + resp +} diff --git a/crates/plugin-wasm/src/vm.rs b/crates/plugin-wasm/src/vm.rs new file mode 100644 index 00000000..8435c775 --- /dev/null +++ b/crates/plugin-wasm/src/vm.rs @@ -0,0 +1,477 @@ +//! `Vm`：单个 wasm 实例 + per-request 驱动状态机。 +//! +//! 一个 `Vm` 包含： +//! +//! - `wasmtime::Store`：宿主状态 + linear memory +//! - `wasmtime::Instance`：实例化后的 hai-process-mix +//! - 缓存的 guest exports（避免每次按名查找） +//! +//! 关键流程（[`Vm::process`]）： +//! +//! 1. `proxy_on_context_create(http_ctx_id, root_id)` +//! 2. `proxy_on_request_headers` → 解析 Action +//! 3. 若 Pause → `drive_until_continue`：循环 await dispatch_http_call 结果 → +//! 调 `proxy_on_http_call_response` → 直至 guest `continue_stream(Request)` 或 `send_local_response` +//! 4. 若 guest 写了 `local_response`：直接返回它（短路 inner.call） +//! 5. 否则：把 ctx 内的 headers 同步回 `SgRequest`，调 `inner.call` +//! 6. `proxy_on_response_headers` / `proxy_on_response_body` / `proxy_on_log` +//! 7. ctx 清理；vm 归池 + +use std::sync::Arc; + +use bytes::Bytes; +use http::{HeaderMap, HeaderValue}; +use http_body_util::BodyExt; +use spacegate_kernel::{SgBody, SgRequest, SgResponse}; +use tracing::{debug, info, warn}; +use wasmtime::{AsContextMut, Instance, Linker, Store, TypedFunc}; + +use crate::abi::Action; +use crate::config::{FailStrategy, WasmPluginShellConfig}; +use crate::engine::shared_engine; +use crate::error::WasmHostError; +use crate::host_fn::register_all; +use crate::host_state::{ContextStage, HostState, HttpCallResult, PseudoHeaders, RequestContext}; + +/// 一次性 Vm（每次 plugin 调用都新建；首版不做池）。 +pub struct Vm { + store: Store, + #[allow(dead_code)] + instance: Instance, + root_id: u32, + next_ctx_id: u32, + dispatch_rx: tokio::sync::mpsc::UnboundedReceiver<(u32, HttpCallResult)>, + fail_strategy: FailStrategy, + fn_on_context_create: TypedFunc<(u32, u32), ()>, + fn_on_vm_start: Option>, + fn_on_configure: TypedFunc<(u32, u32), u32>, + fn_on_request_headers: TypedFunc<(u32, u32, u32), u32>, + fn_on_response_headers: TypedFunc<(u32, u32, u32), u32>, + fn_on_response_body: Option>, + fn_on_http_call_response: TypedFunc<(u32, u32, u32, u32, u32), ()>, + fn_on_log: Option>, + fn_on_done: Option>, + fn_on_delete: Option>, +} + +impl Vm { + /// 创建并启动一个 Vm：实例化 → 缓存 exports → 跑 vm_start/configure。 + pub async fn new(module: &wasmtime::Module, shell_cfg: Arc) -> Result { + let engine = shared_engine(); + let host = HostState::new(shell_cfg.clone()); + let mut store: Store = Store::new(engine, host); + let mut linker: Linker = Linker::new(engine); + let (dispatch_tx, dispatch_rx) = tokio::sync::mpsc::unbounded_channel::<(u32, HttpCallResult)>(); + register_all(&mut linker, dispatch_tx).map_err(|e| WasmHostError::Instantiate(format!("register host fn: {e}")))?; + + // hai_process_mix 是 wasi reactor（_initialize export），但它 imports + // `wasi_snapshot_preview1` 的 environ_get / fd_write / proc_exit / random_get + // 等。我们用占位实现，给基础语义即可——hai 实际只在 log/random/clock 处依赖。 + register_wasi_stubs(&mut linker)?; + + let instance = linker + .instantiate(&mut store, module) + .map_err(|e| WasmHostError::Instantiate(format!("instantiate: {e}")))?; + + // 拿 memory + alloc + let memory = instance + .get_memory(&mut store, "memory") + .ok_or_else(|| WasmHostError::AbiViolation("no `memory` export".into()))?; + store.data_mut().memory = Some(memory); + if let Ok(alloc) = instance.get_typed_func::(&mut store, "proxy_on_memory_allocate") { + store.data_mut().alloc = Some(alloc); + } else { + return Err(WasmHostError::AbiViolation("no `proxy_on_memory_allocate` export".into())); + } + + // 先跑 `_initialize`（wasi reactor） + if let Ok(init) = instance.get_typed_func::<(), ()>(&mut store, "_initialize") { + init.call(&mut store, ()).map_err(|e| WasmHostError::Instantiate(format!("_initialize: {e}")))?; + } + + // 缓存其它 exports + let fn_on_context_create = instance + .get_typed_func::<(u32, u32), ()>(&mut store, "proxy_on_context_create") + .map_err(|e| WasmHostError::AbiViolation(format!("get proxy_on_context_create: {e}")))?; + let fn_on_vm_start = instance.get_typed_func::<(u32, u32), u32>(&mut store, "proxy_on_vm_start").ok(); + let fn_on_configure = instance + .get_typed_func::<(u32, u32), u32>(&mut store, "proxy_on_configure") + .map_err(|e| WasmHostError::AbiViolation(format!("get proxy_on_configure: {e}")))?; + let fn_on_request_headers = instance + .get_typed_func::<(u32, u32, u32), u32>(&mut store, "proxy_on_request_headers") + .map_err(|e| WasmHostError::AbiViolation(format!("get proxy_on_request_headers: {e}")))?; + let fn_on_response_headers = instance + .get_typed_func::<(u32, u32, u32), u32>(&mut store, "proxy_on_response_headers") + .map_err(|e| WasmHostError::AbiViolation(format!("get proxy_on_response_headers: {e}")))?; + let fn_on_response_body = instance.get_typed_func::<(u32, u32, u32), u32>(&mut store, "proxy_on_response_body").ok(); + let fn_on_http_call_response = instance + .get_typed_func::<(u32, u32, u32, u32, u32), ()>(&mut store, "proxy_on_http_call_response") + .map_err(|e| WasmHostError::AbiViolation(format!("get proxy_on_http_call_response: {e}")))?; + let fn_on_log = instance.get_typed_func::(&mut store, "proxy_on_log").ok(); + let fn_on_done = instance.get_typed_func::(&mut store, "proxy_on_done").ok(); + let fn_on_delete = instance.get_typed_func::(&mut store, "proxy_on_delete").ok(); + + let root_id = store.data().root_context_id; + let next_ctx_id = root_id + 1; + let fail_strategy = shell_cfg.fail_strategy; + + let mut vm = Self { + store, + instance, + root_id, + next_ctx_id, + dispatch_rx, + fail_strategy, + fn_on_context_create, + fn_on_vm_start, + fn_on_configure, + fn_on_request_headers, + fn_on_response_headers, + fn_on_response_body, + fn_on_http_call_response, + fn_on_log, + fn_on_done, + fn_on_delete, + }; + + // 启动序：on_context_create(root, 0) → on_vm_start → on_configure + vm.store.data_mut().contexts.insert(root_id, RequestContext::default()); + vm.create_context(root_id, 0)?; + if let Some(ref f) = vm.fn_on_vm_start { + vm.store.data_mut().effective_context = root_id; + let cfg_len = vm.store.data().configuration.len() as u32; + f.call(&mut vm.store, (root_id, cfg_len)) + .map_err(|e| WasmHostError::GuestTrap { hook: "on_vm_start", source: e })?; + } + vm.store.data_mut().effective_context = root_id; + let cfg_len = vm.store.data().configuration.len() as u32; + tracing::info!(target: "spacegate_plugin_wasm", cfg_len, "calling proxy_on_configure"); + let configure_fn = vm.fn_on_configure.clone(); + let ok = configure_fn + .call(&mut vm.store, (root_id, cfg_len)) + .map_err(|e| WasmHostError::GuestTrap { hook: "on_configure", source: e })?; + if ok == 0 { + warn!(target: "spacegate_plugin_wasm", "guest on_configure returned 0 (=invalid config)"); + } + Ok(vm) + } + + fn create_context(&mut self, ctx_id: u32, parent_id: u32) -> Result<(), WasmHostError> { + self.store.data_mut().effective_context = ctx_id; + let f = self.fn_on_context_create.clone(); + f.call(&mut self.store, (ctx_id, parent_id)) + .map_err(|e| WasmHostError::GuestTrap { hook: "on_context_create", source: e })?; + Ok(()) + } + + /// 完整跑一遍：on_request_headers → 可能多次 dispatch → inner.call → on_response_* + pub async fn process(&mut self, req: SgRequest, inner: spacegate_plugin::Inner) -> Result { + let http_ctx_id = self.next_ctx_id; + self.next_ctx_id = self.next_ctx_id.wrapping_add(1); + + // 把请求拆出来：pseudo headers + headers，存进 ctx 之前需要把数据全部 clone 出来 + let (parts, body) = req.into_parts(); + let method = parts.method.clone(); + let uri = parts.uri.clone(); + let version = parts.version; + let path = uri.path_and_query().map(|p| p.to_string()).unwrap_or_else(|| "/".to_string()); + let authority = uri.authority().map(|a| a.to_string()).unwrap_or_else(|| { + parts + .headers + .get(http::header::HOST) + .and_then(|h| h.to_str().ok()) + .unwrap_or("") + .to_string() + }); + let scheme = uri.scheme_str().unwrap_or("http").to_string(); + let mut headers = parts.headers.clone(); + // host 后续要根据 ctx 修改后写回，所以这份是 host 真实状态 + let pseudo = PseudoHeaders { + method: method.as_str().to_string(), + path: path.clone(), + authority: authority.clone(), + scheme, + }; + + // 创建 http context + let root_id = self.root_id; + self.create_context(http_ctx_id, root_id)?; + { + let st = self.store.data_mut(); + let ctx = st.contexts.entry(http_ctx_id).or_default(); + ctx.parent_id = root_id; + ctx.stage = ContextStage::RequestHeaders; + ctx.request_pseudo = pseudo; + ctx.request_headers = headers.clone(); + ctx.continue_requested = false; + st.effective_context = http_ctx_id; + } + + // 调 on_request_headers + let num_headers = (self.store.data().contexts[&http_ctx_id].request_headers.len() + 4) as u32; // +4 for pseudo + let on_req_hdr = self.fn_on_request_headers.clone(); + let action_raw = on_req_hdr + .call(&mut self.store, (http_ctx_id, num_headers, 1 /* end_of_stream=true 简化处理 */)) + .map_err(|e| WasmHostError::GuestTrap { hook: "on_request_headers", source: e })?; + let action = Action::from_u32(action_raw); + debug!(target: "spacegate_plugin_wasm", http_ctx_id, ?action, "on_request_headers returned"); + + // 处理 Pause：等异步回调 + if action == Action::Pause { + self.drive_until_continue(http_ctx_id).await?; + } + // 主流程：driver_until_continue 内部仍是 async（等 mpsc），保留 await。 + + // 检查 local_response + if let Some(local) = self.store.data_mut().contexts.get_mut(&http_ctx_id).and_then(|c| c.local_response.take()) { + info!(target: "spacegate_plugin_wasm", http_ctx_id, status = local.status, "guest local response"); + // 调 on_log + on_done + on_delete + self.invoke_log_done_delete(http_ctx_id); + return Ok(build_local_response(local)); + } + + // 把 ctx 内的 headers 写回 SgRequest + let new_headers = { + let ctx = self.store.data().contexts.get(&http_ctx_id); + ctx.map(|c| (c.request_headers.clone(), c.request_pseudo.clone())).unwrap_or_else(|| (HeaderMap::new(), PseudoHeaders::default())) + }; + headers = new_headers.0; + // 写回 host 真实状态：保留 method、重建 uri（path 可能被 guest 改） + let new_uri = rebuild_uri(&new_headers.1.scheme, &new_headers.1.authority, &new_headers.1.path).unwrap_or(uri); + let mut new_parts = parts; + new_parts.method = new_headers.1.method.parse().unwrap_or(method); + new_parts.uri = new_uri; + new_parts.headers = headers; + new_parts.version = version; + let new_req = SgRequest::from_parts(new_parts, body); + + // inner.call + let resp = inner.call(new_req).await; + + // on_response_headers + let (resp_parts, resp_body) = resp.into_parts(); + let status = resp_parts.status.as_u16(); + let resp_headers = resp_parts.headers.clone(); + { + let st = self.store.data_mut(); + if let Some(ctx) = st.contexts.get_mut(&http_ctx_id) { + ctx.stage = ContextStage::ResponseHeaders; + ctx.response_status = Some(status); + ctx.response_headers = resp_headers.clone(); + ctx.continue_requested = false; + st.effective_context = http_ctx_id; + } + } + let on_resp_hdr = self.fn_on_response_headers.clone(); + let _ = on_resp_hdr + .call(&mut self.store, (http_ctx_id, (resp_headers.len() + 1) as u32, 1)) + .map_err(|e| WasmHostError::GuestTrap { hook: "on_response_headers", source: e })?; + + // on_response_body：把 body dump 一次喂给 guest（首版非流式） + let on_resp_body = self.fn_on_response_body.clone(); + let (final_headers, final_body): (HeaderMap, SgBody) = if let Some(f) = on_resp_body { + let collected = match resp_body.collect().await { + Ok(c) => c.to_bytes(), + Err(_) => Bytes::new(), + }; + let body_size = collected.len() as u32; + { + let st = self.store.data_mut(); + if let Some(ctx) = st.contexts.get_mut(&http_ctx_id) { + ctx.response_body = Some(collected.clone()); + ctx.stage = ContextStage::ResponseBody; + st.effective_context = http_ctx_id; + } + } + let _ = f + .call(&mut self.store, (http_ctx_id, body_size, 1)) + .map_err(|e| WasmHostError::GuestTrap { hook: "on_response_body", source: e })?; + // 取回（guest 可能改过） + let updated_body = self.store.data().contexts.get(&http_ctx_id).and_then(|c| c.response_body.clone()).unwrap_or(collected); + let updated_headers = self.store.data().contexts.get(&http_ctx_id).map(|c| c.response_headers.clone()).unwrap_or(resp_headers); + (updated_headers, SgBody::full(updated_body)) + } else { + (resp_headers, SgBody::new(resp_body)) + }; + + // on_log + on_done + on_delete + self.invoke_log_done_delete(http_ctx_id); + + let mut new_resp_parts = resp_parts; + new_resp_parts.headers = final_headers; + Ok(SgResponse::from_parts(new_resp_parts, final_body)) + } + + /// 在 guest 返回 Pause 之后，不停地 await dispatch_rx 来驱动状态机， + /// 直到 guest `continue_stream(Request)` 或写了 `local_response`。 + async fn drive_until_continue(&mut self, ctx_id: u32) -> Result<(), WasmHostError> { + loop { + // 退出条件 + { + let st = self.store.data(); + let Some(ctx) = st.contexts.get(&ctx_id) else { + return Err(WasmHostError::AbiViolation(format!("ctx {ctx_id} gone"))); + }; + if ctx.local_response.is_some() { + return Ok(()); + } + if ctx.continue_requested && st.pending_calls.is_empty() { + return Ok(()); + } + } + // 等下一个 dispatch 完成 + let Some((token, result)) = self.dispatch_rx.recv().await else { + return Err(WasmHostError::Dispatch("dispatch channel closed".to_string())); + }; + let source_ctx_id = self + .store + .data_mut() + .pending_calls + .remove(&token) + .map(|p| p.source_context_id) + .unwrap_or(ctx_id); + let header_count; + let body_len; + { + let st = self.store.data_mut(); + st.effective_context = source_ctx_id; + if let Some(ctx) = st.contexts.get_mut(&source_ctx_id) { + ctx.last_call_headers = result.headers.clone(); + // 单独存 status：HeaderMap 不接受 `:` key，pseudo_lookup 会读这里。 + ctx.last_call_status = result.status; + ctx.last_call_body = result.body.clone(); + ctx.continue_requested = false; + } + header_count = result.headers.len() as u32 + 1; + body_len = result.body.len() as u32; + } + debug!(target: "spacegate_plugin_wasm", token, source_ctx_id, status = result.status, body_len, "fire proxy_on_http_call_response"); + // 注意：proxy_on_http_call_response 通过 host fn 读取 last_call_*； + // 但 hai 通过 `get_http_call_response_header(":status")` 读 status： + // 我们 lookup_header 时对 HttpCallResponseHeaders 的 `:status` 做特判 + let f = self.fn_on_http_call_response.clone(); + f.call(&mut self.store, (source_ctx_id, token, header_count, body_len, 0)) + .map_err(|e| WasmHostError::GuestTrap { hook: "on_http_call_response", source: e })?; + } + } + + fn invoke_log_done_delete(&mut self, ctx_id: u32) { + self.store.data_mut().effective_context = ctx_id; + if let Some(f) = self.fn_on_log.clone() { + let _ = f.call(&mut self.store, ctx_id); + } + if let Some(f) = self.fn_on_done.clone() { + let _ = f.call(&mut self.store, ctx_id); + } + if let Some(f) = self.fn_on_delete.clone() { + let _ = f.call(&mut self.store, ctx_id); + } + // 清理 ctx + self.store.data_mut().contexts.remove(&ctx_id); + } + + pub fn fail_strategy(&self) -> FailStrategy { + self.fail_strategy + } +} + +fn rebuild_uri(scheme: &str, authority: &str, path: &str) -> Option { + let mut s = String::new(); + if !scheme.is_empty() && !authority.is_empty() { + s.push_str(scheme); + s.push_str("://"); + s.push_str(authority); + } + if !path.is_empty() { + s.push_str(path); + } else { + s.push('/'); + } + s.parse().ok() +} + +fn build_local_response(local: crate::host_state::LocalResponse) -> SgResponse { + let mut resp = SgResponse::new(SgBody::full(local.body)); + *resp.status_mut() = http::StatusCode::from_u16(local.status).unwrap_or(http::StatusCode::OK); + for (k, v) in local.headers.iter() { + resp.headers_mut().insert(k, v.clone()); + } + resp +} + +/// 占位的 wasi_snapshot_preview1 hostcall：满足 hai_process_mix 的 _initialize 链接需求。 +/// +/// - random_get / clock_time_get：用 host 端真实实现 +/// - environ_get / environ_sizes_get / fd_write / proc_exit：写到日志或返回 0 +fn register_wasi_stubs(linker: &mut Linker) -> Result<(), wasmtime::Error> { + // random_get(ptr, len) -> errno + linker.func_wrap( + "wasi_snapshot_preview1", + "random_get", + |mut caller: wasmtime::Caller<'_, HostState>, ptr: i32, len: i32| -> i32 { + let mem = match crate::abi::MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return 1, + }; + let mut buf = vec![0u8; len.max(0) as usize]; + // 简化：使用 SystemTime 的 nanos 做种子异或填充；不是密码学安全，对 hai 足够（hai 几乎不用 random） + let seed = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0); + for (i, b) in buf.iter_mut().enumerate() { + *b = ((seed >> (i % 56)) as u8) ^ (i as u8); + } + let _ = mem.write_bytes(caller.as_context_mut(), ptr as u32, &buf); + 0 + }, + )?; + // clock_time_get(clock_id, precision, *result) -> errno + linker.func_wrap( + "wasi_snapshot_preview1", + "clock_time_get", + |mut caller: wasmtime::Caller<'_, HostState>, _clock_id: i32, _prec: i64, return_ptr: i32| -> i32 { + let mem = match crate::abi::MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return 1, + }; + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0); + let _ = mem.write_u64(caller.as_context_mut(), return_ptr as u32, nanos); + 0 + }, + )?; + // environ_get(*environ, *environ_buf) -> errno + linker.func_wrap("wasi_snapshot_preview1", "environ_get", |_c: wasmtime::Caller<'_, HostState>, _a: i32, _b: i32| -> i32 { 0 })?; + // environ_sizes_get(*environc, *environ_buf_size) -> errno + linker.func_wrap( + "wasi_snapshot_preview1", + "environ_sizes_get", + |mut caller: wasmtime::Caller<'_, HostState>, count_ptr: i32, buf_ptr: i32| -> i32 { + let mem = match crate::abi::MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return 1, + }; + let _ = mem.write_u32(caller.as_context_mut(), count_ptr as u32, 0); + let _ = mem.write_u32(caller.as_context_mut(), buf_ptr as u32, 0); + 0 + }, + )?; + // fd_write(fd, *iovs, iovs_len, *nwritten) -> errno —— hai 用 println 时会走这条，简单丢弃 + linker.func_wrap( + "wasi_snapshot_preview1", + "fd_write", + |mut caller: wasmtime::Caller<'_, HostState>, _fd: i32, _iovs: i32, _iovs_len: i32, nwritten_ptr: i32| -> i32 { + let mem = match crate::abi::MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return 1, + }; + let _ = mem.write_u32(caller.as_context_mut(), nwritten_ptr as u32, 0); + 0 + }, + )?; + linker.func_wrap("wasi_snapshot_preview1", "proc_exit", |_c: wasmtime::Caller<'_, HostState>, _code: i32| {})?; + Ok(()) +} diff --git a/crates/shell/Cargo.toml b/crates/shell/Cargo.toml index 358d7f61..37098da6 100644 --- a/crates/shell/Cargo.toml +++ b/crates/shell/Cargo.toml @@ -52,8 +52,10 @@ plugin-set-version = ["spacegate-plugin/set-version"] plugin-east-west-traffic-white-list = [ "spacegate-plugin/east-west-traffic-white-list", ] +plugin-wasm = ["dep:spacegate-plugin-wasm"] [dependencies] +spacegate-plugin-wasm = { workspace = true, optional = true } spacegate-kernel = { workspace = true, features = ["reload"] } spacegate-plugin = { workspace = true, features = ["schema"] } spacegate-config = { workspace = true } diff --git a/crates/shell/src/lib.rs b/crates/shell/src/lib.rs index b60f6442..8bd47644 100644 --- a/crates/shell/src/lib.rs +++ b/crates/shell/src/lib.rs @@ -118,6 +118,9 @@ where { info!("Spacegate Meta Info: {:?}", Meta::new()); info!("Starting gateway..."); + // 启用 `plugin-wasm` 时注册 `CODE = "wasm"`；注册放在 shell 而非 `spacegate-plugin`，避免与 `plugin-wasm` crate 循环依赖。 + #[cfg(feature = "plugin-wasm")] + spacegate_plugin_wasm::register(spacegate_plugin::PluginRepository::global()); config::startup_with_shutdown_signal(config, ctrl_c_cancel_token()).await } diff --git a/resource/hai-wasm-demo/config.json b/resource/hai-wasm-demo/config.json new file mode 100644 index 00000000..20d4e030 --- /dev/null +++ b/resource/hai-wasm-demo/config.json @@ -0,0 +1,5 @@ +{ + "gateways": {}, + "plugins": {}, + "api_port": 19876 +} diff --git a/resource/hai-wasm-demo/gateway/hai-demo/config.json b/resource/hai-wasm-demo/gateway/hai-demo/config.json new file mode 100644 index 00000000..c0b15c81 --- /dev/null +++ b/resource/hai-wasm-demo/gateway/hai-demo/config.json @@ -0,0 +1,16 @@ +{ + "gateway": { + "name": "hai-demo", + "parameters": {}, + "listeners": [ + { + "name": "http", + "ip": "127.0.0.1", + "port": 18080, + "protocol": { + "type": "http" + } + } + ] + } +} diff --git a/resource/hai-wasm-demo/gateway/hai-demo/route/demo.json b/resource/hai-wasm-demo/gateway/hai-demo/route/demo.json new file mode 100644 index 00000000..17448cef --- /dev/null +++ b/resource/hai-wasm-demo/gateway/hai-demo/route/demo.json @@ -0,0 +1,33 @@ +{ + "route_name": "demo", + "rules": [ + { + "matches": [ + { + "path": { + "kind": "Prefix", + "value": "/" + } + } + ], + "plugins": [ + { + "code": "wasm", + "kind": "named", + "name": "hai-mix" + } + ], + "backends": [ + { + "host": { + "kind": "Host", + "host": "127.0.0.1" + }, + "port": 18099, + "weight": 1 + } + ] + } + ], + "priority": 0 +} diff --git a/resource/hai-wasm-demo/mock_backends.py b/resource/hai-wasm-demo/mock_backends.py new file mode 100644 index 00000000..3a36bc26 --- /dev/null +++ b/resource/hai-wasm-demo/mock_backends.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""一个进程内启动三个 mock HTTP 服务，配合 hai-process-mix.wasm 联调： + +- 18091 ac-service：API Key 鉴权（返回 ApiKeyRecord JSON） +- 18092 asset-service：资产查询（返回 AssetRecord JSON） +- 18099 upstream-echo：扮演 hai-gw-server / 任意业务上游，回声请求头与方法 + +每个服务都在独立线程内跑标准库 http.server.HTTPServer，无第三方依赖。 +启动方式：python3 mock_backends.py +""" + +import json +import threading +from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler, HTTPServer + + +# 简单的 API Key → ApiKeyRecord 字典（按 demo 需要可继续扩） +API_KEYS = { + "demo-key": { + "app_id": "demo-app", + # 包含 demo-asset 在内，hai 才会放行 + "asset_ids": ["demo-asset"], + "allow_ips": [], + "deny_ips": [], + "allow_mac_addrs": [], + "deny_mac_addrs": [], + # ISO 8601 UTC，预留够久 + "expired_at": (datetime.now(tz=timezone.utc) + timedelta(days=3650)).strftime("%Y-%m-%dT%H:%M:%SZ"), + } +} + +ASSETS = { + "demo-asset": { + "asset_id": "demo-asset", + "asset_type": "tool", + "asset_status": "published", + # 让 hai 走"分支 A 转发"：写 Hai-Upstream-URL 等头，路由到 backend + "runtime_endpoint": "http://upstream-echo.demo/echo", + "runtime_endpoint_method": ["POST"], + "asset_content": None, + "asset_url": None, + "max_concurrent": 16, + "timeout_sec": 30, + "qps_limit": 100, + "asset_secret_params": [], + "asset_secret_values": {}, + "allowed_output_targets": [], + } +} + + +class AcHandler(BaseHTTPRequestHandler): + # 静默日志（避免刷屏，可改成 print 来调试） + def log_message(self, format, *args): + return + + def do_GET(self): + if self.path != "/ai-agent/internal/v1/ac/auth": + self.send_response(404) + self.end_headers() + return + # hai 把 API Key 放到 hai-api-key 请求头 + api_key = self.headers.get("hai-api-key") or self.headers.get("Hai-Api-Key") + rec = API_KEYS.get((api_key or "").strip()) + if not rec: + self.send_response(401) + self.send_header("Content-Type", "application/json") + body = json.dumps({"code": "invalid_api_key", "message": "unknown key"}).encode() + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + body = json.dumps(rec).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class AssetHandler(BaseHTTPRequestHandler): + def log_message(self, format, *args): + return + + def do_GET(self): + # 路径形如 /ai-agent/internal/v1/am/assets/ + prefix = "/ai-agent/internal/v1/am/assets/" + if not self.path.startswith(prefix): + self.send_response(404) + self.end_headers() + return + asset_id = self.path[len(prefix):].split("?", 1)[0].split("/", 1)[0] + rec = ASSETS.get(asset_id) + if not rec: + self.send_response(404) + self.end_headers() + return + body = json.dumps(rec).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class EchoHandler(BaseHTTPRequestHandler): + def log_message(self, format, *args): + return + + def _echo(self): + # 回声：把请求头（重点是 Hai-* / x-*）放在响应 JSON 内，便于断言注入 + seen_headers = {k.lower(): v for k, v in self.headers.items()} + payload = { + "method": self.command, + "path": self.path, + "headers": seen_headers, + } + body = json.dumps(payload).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("X-Upstream-Echo", "ok") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + self._echo() + + def do_POST(self): + # 排掉 body（不影响 echo） + _ = self.rfile.read(int(self.headers.get("Content-Length") or 0) or 0) + self._echo() + + +def serve(port, handler): + httpd = HTTPServer(("127.0.0.1", port), handler) + print(f"[mock] listen 127.0.0.1:{port} ({handler.__name__})") + httpd.serve_forever() + + +def main(): + threads = [ + threading.Thread(target=serve, args=(18091, AcHandler), daemon=True), + threading.Thread(target=serve, args=(18092, AssetHandler), daemon=True), + threading.Thread(target=serve, args=(18099, EchoHandler), daemon=True), + ] + for t in threads: + t.start() + print("[mock] all three services up; Ctrl-C to stop") + for t in threads: + t.join() + + +if __name__ == "__main__": + main() diff --git a/resource/hai-wasm-demo/plugin/wasm.hai-mix.json b/resource/hai-wasm-demo/plugin/wasm.hai-mix.json new file mode 100644 index 00000000..e898942d --- /dev/null +++ b/resource/hai-wasm-demo/plugin/wasm.hai-mix.json @@ -0,0 +1,18 @@ +{ + "url": "/Users/sh.zhang/Workspace/huayun/jiyan/gateway/spacegate/resource/wasm/hai_process_mix.wasm", + "validate_on_create": false, + "fail_strategy": "fail_close", + "plugin_config": { + "ac_service_host": "ac-service.static", + "ac_service_port": 18091, + "asset_service_host": "asset-service.static", + "asset_service_port": 18092, + "ac_auth_path": "/ai-agent/internal/v1/ac/auth", + "asset_lookup_path": "/ai-agent/internal/v1/am/assets/{asset_id}", + "model_error_keywords": ["error", "ERR_", "FAILED"] + }, + "clusters": { + "outbound|18091||ac-service.static": "http://127.0.0.1:18091", + "outbound|18092||asset-service.static": "http://127.0.0.1:18092" + } +} diff --git a/resource/wasm/hai_process_mix.wasm b/resource/wasm/hai_process_mix.wasm new file mode 100755 index 0000000000000000000000000000000000000000..ddf0fdfff59293e96da2b8431b33a52da5e03b71 GIT binary patch literal 984275 zcmdqK3%p%beeXLTYp%!MYwx}C05_7J&b3V2O${1bPQpRXZDy__9_f{~oO?L;cy6F1 z3OgZ$5D|MQ*+9Ukh_OaRMY~m$sHka+7OmBNYt+x#ELAJ3M3)<3)i{NFATT-reS&evxC#_ziO&9B(C z9RwFO{C5(|nuCjm@?t!%1DK*xz4&^{!hr#;PwLHe>aSkZ-RoT6ue1KDL!tfw1>}kh z0nxSUNdae2?T~88!E!ZHETGaVd_tBX;kxS_)Oi`bQ?2@usa; zpS*4R71z9U!;3cU*zo%=e&Gu@ZQt`Gj`apQ(dH(k7uwl2ST+lH$z-tyv2u05YYz*I79yZEy0R|TP}o;$91+m6dF z*}Q>4xa^9J8!o?W%a+To+C+65uW~Rg@r`facFBfIF5Y^{rY$brS0Gp~dGYq`o3^T= z%QtPX)U3l$IwWS~p>~(f({A&H8#laU z`(-;e1%vbCPzNvBu<4r1b_79u=fFD$mIciwS}JKKP5sxbCrMq8dQz`9>v27f<2lmn zaZ*o~*Xt+M>nA^HARZhY4XJ^ml;VH>CP5I@qa>~m22s=q;(F4kN6mN%ZPoezlqd?L zW)y~D&}cL$RQwqY>Z8Gv`4gYoXoigjul2fe3<642Z$^Ts8P`uaMbGFYst@&{5l6h9 z;_9ad{7Gm@^~O;YB}oKqA$8Sb63NE5IEa&GgFedH+g_!9;ci3g)) z(J%ip9b3{Ijgu$_7|<8h`4C0HX#EtGh#K`K^=SC0GXz(FsTS+5mdI61g8rnVK@v0Q zje4yM%9ji_z?os6{L~c%15r50fP_(FaA4W8X4GmV`f!Q>)_-9PKtNIt7>?L#4MT7q zHAj0-Ngy6_@Mka-)zAm9HP{>+q|z9aM|J*N5!2ho99V45fyM?MrYWm5Zt9nM83pw) zN(SoTKs|{Df_O=<5v-)|zw)a?gM$Of&=8-RL(O<_s2(v6jIw$U_;mV#TXz7&E__TO5Nk;3V$!HXgLf;KTFFD8tj_IWlR0PEaqr$`} zMUAZN!7vA5l7y$65}s23_0CgQ366m{K0O9or<^i6O5a9LAw|Qx60AX?G5=VPM@fXI zTBEHw{v*%tG z{>00I1eEb-sPVL*J|rHFn?vv~9Hxr&$7lh@)gJ(+I^Q5Ho-jZ(SAV*i;e)=?1cU~3 z3><%hWy_*y>0q!dS}lqd=|v5=Le-EdhJ5PuU9f1pVi+id6{96lPnGi@-<}cw#%~Ii z{y&L`l#x5*40;Ab3{Wqk2*Ms%{vShNRlV>>eGSjGXF>K6B zsL_9NRflIi^OHB> zcuR;fmb`t_C7ZA|g1Z=(mpPez}LJ{0|V^p)sv^o8h4(S6Z$^u6dv^sVUI(Kn(yqko8g96b>I zbM(XLN6}BBe~ccCz8<|R`a$&N=&R8?qhrxeqwhyYqkoEi7JV(cKl)_t@BGGp z`OW7&z&$%Of z`s)Ale?0pIzxChN|37D6aPF#KKW*&J@L+Ufb5Hz+`1SGY;+vX(9)CE#C;ma}hpm^@ zz7+3j?ykMOc3=GQ_>1xNwO7=B9#7PM5C$@`P{B>R$gC+|$&m)xCvB>A)C z!^xi}A4)!&d@%Wonf!V3m&yL*1Iazf$C7s>Z%=Md-kVG%A5Z=$ zd42MRWN&g?^4jEe$s3bh^_%K9)^D%Bs{XqAYwLUKd+NLEudlzR{>J(n>aVW(m zSJZE<-%`J${?7V4>hG?i^LCX8k+$BlRED_cm^5 zysq*3#%mh~8-LySOyh4F_cs2v@!7`b8iyL6Z~R^3?;D33pK5%e@x{iM8uvB6-1ti4 ztBoHwjy1m3nrJ@K_;%~&<`-MPXgt#TdE;Lj*EiqX{Kw`!&A)2Cs`<(0r<r0OuQ$Kg{895;%^x&B-~9XLvF2Nv|Ipms`bqPjn?G(o*nC@a zviVT+;pWeq-*0}q`QGN8&3(=HH1BG@yZMgh1I-^czt;Sh=0xik%}1I?n*Z9op>=)h zaP#k)-)#PA>rYzWZ60lYr}@3+mzrN_zNz`f=C0O_txvV?Zr#)RQ0vcHA8dWI_2t%` z0|#609eCfs=URU;@Y&WU2POtSG4PYt=Ubl|c(C=;)<*{hKKiy_edhqG<-di|&ZkW9 zC&)URd*Mp^8LjL5(@iAxqHOS-p&$*@Xie14_DoEX?Wt*)wa2BVc$xSG|8_q33Nie%uj-?9X>BYDvg;w9 zYn}tW)b!prq_uKLr~8^B^J!7AHVV=pYmQTIbDXNy@Lnqi`{F(r3St6=oSeR9-8?Qi z9qVhmLQeB`(;7IDZcaQhRpE@s)Xi|Q-Q3gHcBPxgN96Rv5IH$t3-cn<(aX0MPcQrX zntDu#Ea~OQ!n!%t*Y*MsSq{|#5qW5_Ekp$|EU<;?`C8c7$5oN%>?jL5*Wc{8Iuch) zBN$glDerw;9hF>ttgjUYyUNv3$<{&bz z|6{I^h}KEIb#BmD^g^d78T_iEtl)pLuhj~c=VSSjQWwYz`}*m>le+7^RWS>x==RQ)7NT+ z4d$yGB3)?Rlk?R*AB&abxj+Wk-`7Hg#pWZA_(aHKs;`v=k>~&H6Hsw(%{v3E^MEN+ zKUaWYsv;^bRm7pbcH>eJ%r`dRZ=qu|-B)*I+~@c~H@aGA?R(~{eLku!jid(`W+YAawNRnjd4-Cz1%Gk^wqRdVkCQF<(89Vo)z@~V zoAV*E)UXRgpz}>XN{g`t`&wC$fSH$6oGo|)q%v`1m8%v+ zD)R#2Y{3%%V$b4%m`@nW7W}qR>hW#CzE&&3@c5LvKwjA2*TMpLp`_FWwlKBW7D{Ei zKwdaBUkmf`LU+8 zwZaDT)h%0afpt&LSND7@R+8re8DM{33l$cdk38ZNA&;rPRu)8_^I=cc+5)k6sIS$^ z$n^Kk+nM-_qt4g88jAZIjd99*pG%@*Z%+5M5~Oj}B@ve{iFqxqT5jjMX=djeCVpc# z6|bdn)y}OrIFI$UTXADn87tk~D?Kjl@y>TjHQUJwN2Iqth0Woho1e|=cm@&MucXpUE#>V4M(f(Lhb78TyyW)+B93B;ZmK;X@7 zIv_gEn~Rz69lC{s7u0*WY9(CD3%fH-(t7860%P#Mm$0%r;Sm{*au#P z(q3Awd-M%ObG-9|%24#;Y+~;Z?~jEC8;m|YR0d;zUqcYvny8tIpW^1)m}lA1-|6d9 z%e&oLAVv7ccTV@yM%wKBP;jKJv8dfuRs5iF{NyQ*TG}e-PxE8x9~OqCsXpKslA?M47(A~wA8*^vDFllfzPz?331jRvb?4z4XFOkx7&1eS~@ZtjUjP#3Yrl3Pmg&UH`g#5_ zw4aw1qn_tp8yi3X8^GPB+NS!P7j{|5XTk|#C-8C6Sk~*{P?2XRVh%3qxs26alEs3 zT_cZ8nGP$_OVPTy{-(iLG#&SFWHvGDAK_2q`DWI~SvW3pU2#oV>JJCwolm@6CJ|W= zhFvDM%y;;(H!MErQ#0G?ipkg?O~ccoPPc}E!P_mkDbm{7=zNN5G8epWeR^{*;q6c` z{=Qy)*V=fJEZGgkHwtFAgyZky%We8HHlBS<5mx-Xjb{JS;Gs_5M=h6?zp*Ia;6uAC ze#UtAvp3BwPX7-^%q7CHC{?eS6Ql7TB1dyF%eFWpthkf(!zgg4y(I z8I5e9^KQj^kxpma*Uk;}l9aLN{GqU28??d$OqyJgG^Wry)rDsdfuGZZR`&Gg3?=Qx zU>Xg9{cbA@x>0*@kjDU4cEs#v;JK&vct3Y2B(ZHD!tM|i4`P^hK1R<85$!TJ$imJ? z^wt}C_7GFF?ttRZh#=al&J3-I_UJj_#zeSWNKa^Pn5NX9EL1l<2(hJIn18$=9oXEX zxiJlRd-HD_vKYm*EwBLW?E*~S_W-+|d&6#9*ik3ap%i+H(}D4H@ZkZApHso{9y6M~ zi~v)gxoU3p*K9TBO{h>&b2Dv~)wX=KbBx+BSfruzhZMo5u7E-+iqc`G+|0iEI~Mgf zKeJ;|(R`c^!!#a(>>#AlP9qowe27+#p9R?q(=dg>FW|u&Vkwm{+T{e!q&TFJN-CM`nF_|Uo8LS&!AMnw z4O;NALg&xbAZetlT6HriLoe20Eh zXj0!Kf{&fKHd60MMr^G?u=DF1@T%+M`=yU0ImEtn@J#)4w*GmR{yA4d@^GYo?}G&O ze8}D&j4~>pxaOQwmG*Gd`K}~kR&zWh?hx4+=}^tyuqXa>1IvilcJ!jG=H=IbMi%bA zvO6qNhuC;g_-!k@4*D|kAAmT9x-oJnl0zeA&Y!#Zc&QaHC<)%j%6GhRF~iq-fM}@9dX?fI*~S{3|nBuqCzNdT_NtC#pg<6d^@A~cIK>a zW5u^sv%ak;zNNFiwTo|^$~Ukx9gaH()*y~ZFj><-*m0e|m!=&2WG!s&6uXpvJ>$U9 z6mP2Mz>^p|Eh*6E_HJGJuBjLDMy5iWmTQ}GVe(|{9W!#p%EgDJI}p{!0u*u`ZBE6H zAZqIvYTX9QAK{fl>Pj790Q^}*BELKOjbPR9iQ02d?P9WuB@vehml_465%qY~ZaZ`c zv4)FYtnMb@_@d`NSvW!rWjn6|hO>gUgM~rK!n1;5i8nFL2tTMLADb{A8=$YvC{Mj= zFPnZn{M7kdZ%jp9Ns}y~1y$wOCwM?i5}3i@O2$`&Mdx?)c^;&GB zT5d^PKq`jBd1bzH`AG%O;)6h%Kx&>%Guo%HCfdUDOf7SrJ>D?`!uCDDW!etWg;14E z+y!REwTw#VuYlZMo88#i0b+%$QQViGl19yhppQ=zTge6!#K)*{j4Bx`Z&rY5$>~DOU_&&L(A}HN2{q!l;T1Q z#^nOwZ_tkD;h+iSd#O)vn3MclF(-M$octpF39RH(5o`Z4s0CPjc-Ao$=ac5ySXt%9zu4m53 z%==cER`&IcS(bF586Rck;QYfLo|E&cp)=&gVI_GAsrSnD!lZTSK0HXxG$@R{BCN};|3w&TtbBt3?h$s0bs z);=wR%A>S#D%1PQccXdK56&RKWR4QDUcNq9f3&ea=zK;B$9huZyoB|^@2W4S#hq>& zVAt=&kwO{dt28F=g=~3eT9OKojEvfV0Pz}Tn{FEBsd%{TY)IacikbR22Z7Kw%+x2g zk){*xvQ7+>nD?)#{xRs!26R$$8bh1I_e|-kLi5~;&=jOjYW8m~GWuCaMxnT@9okz! zA;Ti`OC(n}cFf*wf(57~$r2bb8zlY1>ZGN+pV3oKuJykY^ z0WLKJD3D$L7o>g0inV7F1tXMw(LN1`MxUjB&aF~K2w2FfiP)1F;NeKL5+nMsi@Dxd zv^yUOqqJCDEwx5}WcAJf`q_e%YR(FdQaFvz3ck-Xy9d|m?&~FNf^4Gx%GHM|+srf1)G zX4f_JL$62O>o)SeEYIgGEZIw-r6^j1K~HNiPhtyZD`3WOpdtJ9j3Pj!lN&pva4B*Q_#JyUr}zKaFPrO(4Sz z2Gij2>*Kg`Ofxv+Ow`_BQn?-S&`dBlc_17F%UCqlGt-M-t1z?juhcM=PruS&@Bl}J zFxL_R2JFf*T7iE^vBD(FM#~Gkj2F3z`L=P^w=KoDZIy3e4Ws4fqD)uC87^CxE!mP? zb0uyW)LnPHD7!k1HfNV2KC`Bu!3x0hi|_dTvSbIgoSWWyQ~&fJH}xOPP5t{TrvBx9 zArw(*#aZXgFAg3R)#XgphhHZhATvnY!)@~MDv5A(E?P0nzT>{Ti9{FwQ5k3iV!yc} z8ZOPoW94!VQ?bY)p&}5dPOf~IHO|*$=hg*cr5C>KW6XBubk`NDg?5Ij$# zM#VM6ORbv-Myl8ock6{BL^&2_c_LFpCV=Wr<8C9SXc}qKjJzMm?u>FsG|THX&GhFB zPj3e>V|T7ILlLz!CE(P~H>c_%EOw%%Mba6i^O$DDRHD?Kz=DkJfAf$DkaX1!cR0QUIR-lm}uK9||ak zawuyRC}jhQIHuz6@SIS#Wtgxe=IC{37L>y!lmd9MP=?u2XLJDs{sAGK)(Tf+KDrgMGI$2daA_4HY^uhOokXFkTB3c<7b2?d`(KLHz$Q|`eW zH4i2g*~kI#av=)^XRGElJ?~}pJQl6wF6gFAP3??A6*Tk7Gj^g>AVoEhPQK59Hs(=o zwTX)Jm8H&28c@;;tEz{^D=1}H?F-W|9dV`fKFRxBmKMYvX8T`-Hnw#cdA?Ye$vt1P zGf3G|rd*o6XYZ6XD>poYl^t~}GoANg#{rHOg?o^u+3Ofx#f-YmVomLx>Q?y4<~P3L=QjkhO&q7-<&;!lK@_e_2Ar_aK_#9^+%aU*Rnu(W&47HIrO*>v#QDu)>4|anE1^GP#40PH z<(S**K;M|xp>HB+^Ev2Ze}fg)f^hU*TFi>waJX4KD_hMbDlm4q4E!t{It**8oDzZJyk19Lq8wT{bs_b!I_Zhn19nUBJI#)j2# zkocH%jz){H zn0oqRxZ$yk!tI93WgeHTk}i%!cjr<9m0)&^$gsfWy-wzU%aWG)?J4(B@Ou9ZPYhls zp7wa*^cwQ}w*TL~Riov!P>lg~Dn z`?ch5Xeu9}#eM-5QUj*)a~+{}(DxX!n$O`)He-ys+HL|au5Oo$%W7ps-b(t|TN$0V zm9e~)Rgb-umGic8M&8PqkG+-E^R}`sZ{=B!y_K`)Y-RAv%{amZk>oL@Q><>r&h5d_ z;?xMTeXo`zZRFInYMt@JNY-vs#?&KJZog`8+iG-y!~RL#w{uAoe(2hV_Y zm`k}brw6ND8S<@SUZ8^Jc-F5~J(tjHy6(V}sZ}~)HOkRDfb7H%&P0aU;i>rYaQL3u znKEmY8frL9aQ%Y=__S+L1hnp_$sz4g&aZDYpFLF9)JeV(g%6~4zqn%)G4Pt(ftB?U zW>MAhOro-*M?ZgKgY^-x?+E2@6thOu>$kKRA~-5UkK3Q(9y5MOwd z{49Xv1i>^z9b-r!i(YbPujR%U9UOJ0HH;}ztwqnvzEZ|^*0KhDMq#9B@Tqi)ZuK9@ z_QYgmka82VZ$agWX>Ad;8a4q5MUZ2#LO`M{=VXNv2!V9sWAI(Ia3~im8U^5t_F6tFwmV?l)XJ>TfW@;6#&U8ex!n<+C=G|9t0_iSM-q_)T8$$}l}J6b9e#!3aaDmY)4`FXEp?2ot-mc11z69w zKym7adx<$Pckz$?qS>Q$rKx%>EV&9s&4B^106L?m;XFOXe=}E4^M1%meUyG^Y(R9{ z+T0_~8Of1P|I6;hxGLzgizpZ-q5uS&50u3T5TlPiz@Rub^&y3FJw>d}idPw2qp^&P zP?~HNEw(NIV7_%>j71gaCf1rPc+Q=MILj!BlNl$smi$Ce`w53GoQ4V0`8)&_X;`?4 zsiQmKs7Wp_b{s=i-bdWzCyrFep>kj+!|KCJb$}}{49FZBF!weARBNIaPzid3RU^G! z09dx9ZcF77P|!i!hs@*RdI6C?v2+Q)FNE}|bPKft5zesOS3Ls+6k$II-Vsw8X0C06 z1OanQ@((zi^jGD)B(n6(#guqU4|S6>weK47YU91SxAXw;f*$Jwk_>_pbb1kBb6vkr z#3OtnDs%@$a4)SV5PlmErfiq;(Aam@#uue9tfIkqI0Ax>YP?M-yh^Fh46ljS5zDN0 zwOqtd9P&QU8nojj4>3Hur6`I3dxpCrS0{=%%`coWsK73T?u0wz>uUB4d3tI*RGVXi zO&XjQ2kWEX0R#fR<@D*i#-TLZpT{xDufQh~{9<|@zZ)xl7bXm|gKipoe;(OnbNs`4 zTN_V>sjocyvT3}bVlqm}(!JuR&YBcGgo6hW5oY(hNzt+VU7K(CepC*iIh-!jN`H74 z8g5PW07EA?*-<@x%-9ir+Y;pztBvw>=WCSmc599DP$l!d`b|X#^$na%O#Rkc=jJA! zGsf+5Od0NC<;c)eEK!PG%fI5*ZCOihMt*I{zH2D(vL?rR>FqukvH)fW7Kmkuu#Twy zG~qCw;q7J)GRW7jnnnCBF36~wV0*>0 z70k!@oCXeczCdU?Wx#QEDDKjo4h| z0T!80Y5*8KkqYGTfS#T_jI2mIQc0U8+TzBkiJXGJUG@3(hdl0pEiYG_>UYo0z+sKYnI>;4qJjF=RTgaA_26j zM1d|tD=tLw)S`IopKk(&wZeI;+3Rq_ri*{aG` zUvk>@MDwYo5+^32-BF&j)Wi;u`NW((YAM?!tM|?;fj?S*%I@`# z1UIg~{)Sz<_e@Srgx7UPu)sQ6&ZEZ!r;VQ$1idBc$W6Uy{mz>x@yNgZ{AWM<#>XGI z?%EsIPvqrx-q_O?JQJwhrB-$+JI1he|KQFnzM5*%;VU<@NujrFytf>wE1iJ!t);xw z(xtiisnw_TJFiV6)kin5L6@XU)`vIuwZ#|w_Gt$x+QW1hrYmi8TLChaj=~3C5-Ye` zNjL#e2Y!wXI+)K2N&ISdm)fu^1MUMB&dOXhH^f1ADb}H;)JyeGJ6*;E1vbSD-UFee zOWB^OQTY{(kIf=tZ^RSIK>%tobssi$@DYtl?kz$3>8g^Lj zE?|+QttCmOKaQGCN|&a!e2H3GuG7E+*_*WgvH|$S43FLn;>xMObd8hYMZd2 zCMoK6dXkJmdkc7Yt=Oi?Hz>_D4O=uMB?s2V(h@006bVUr?6kCrb`ZiY z!k<)TcDF;wBJ|#;Z!55!&DMMJ1VP;31fZ`me`=e}lVdDuACb#MN3%fTj|JpA77g+n z-uUPsgBI{V6;^bH)Th%K{o$OFFugdnXaPp>?2^B7A)}`T_qH;M3_S0RHGm3X_=G0I z$^dQDG+fyv`pK*wGVudk8HH;2_r@#-YcFVtFtoMMz8KYYz+r3=*2E7h)`VK?w1Q#7>UYyW*+{L$kvs@TZs+FsZ8ZS`~Nt%75UnLaenHi~w+Ts3aGr z-aUGHQ}1q{w!e~gS1o(ZtEPl=8z{p>t!n;hHpWL5p$R^tAit04_ddjME<2zrrN6S$ zt$k&;vRf+Ycer#DAO#(o)d(pss-s61-O(Rk|7abRnL;PPKU;(=9rc|y1XMenFCSMr z=R0j<{HjX2@AS0lzqOJ+`M4qD!9^qEZ?z2yc*LHfJEoloUq1&a2tFr#J?A0*2s`YhZGMn}J&)a=2tciwvh`I9w z_ec{3fs4zfFjrE&50nZ!(L0b>)W)&6Ybx}?INLTpJQUa5pNL1OEppa*c5AsA*qOE! zItF4(XUB$1U)`TRmZ!U|)J6U@Pfz>vv+df|Guof7tqmF64Fcq#D(W$Du5>#Ru@~7F z3G`_M`cx}k^(}%*6 znbaFK*A&gZG}~=EW=3<{(SqHUWRg{%T0cj!c^d4WwW}R6UVhT=PqB` zbJG|PlbS7I(wpj$LzIqT>Eiiw=2fQSR|sX4>$pN1PJ+Kd-FKuqX(kxO$D%tbW|$eB z5WIy3W24#6+bS8H|A|0&L)Bbhp)UlL#%SjLqM}+1@zV4YsZF3?Ch;(N=q5_&w0EJp z)0oWg#BDZ^jX62db?}bTN~nlpdC=^z4nWyo=13)b_UllaNtm(_+?vl<=Y+@ew@u3f zmp#SL>{L>iyo+h`UE*c>S12|VYA(-bS{&wwDDxH}PE#g;weM_E?vxu|F*RXif+!Od z8W`RScM3^W6*PH*fK57IUYjc5g+8ExV+V~5An51KxiMv+Il*+h znu^xBziD`SFdYx;Vq8q8*xY~`wc7U1JCx6OP;y+30V6S7UP6pH(F+Sr`QIX*61*<6 z{j|*N-7j`Ngc&9zfZT29C`{zLIJCk*X9KK(uJHhO>!qVO+QMl10iAHA+ecbx;f07q6ft1n&)?YC*9!r_Hu(=B;F{g9VCed%tEj@nM&xZLT> z3bJN*ak7~AH{^B(lXZiyFuLsI=aoKjo0U^Ons zw!9@V8U=zWcbOTn9E4Ea@?5reFG$MP3Mt6*UG{db{>?7}WFe>AHf+Q?J-45&Z{zSR z{FuX_mJKLAp_c7+9I2&k!w)-qTnf~Uk|=Qmsm_HX25FyB*Il8cXF$Gp zc1J5`(y61i7{9VRge9S_PR=ec*^ikKhZ(nrVMVv2Qls4^n4mF!Jw1Z%G9@nWF4v>e zJqd>4hyakxzn$BiL_RHE-91^k&+I;l?`9sSgNKalHuT~|yI0i{W zz%-g(7kh<5L;s+m-;U@Y>qOWc^@2fR5O9*mKTEc3;>+>hQj{JZ+Y zjZBGucIWitKh&yjXg*W?uC|iQ+LV5Au=;uK*QgGRKuNA8CY~n(dAFV;Yi+ld_P$K& zeb@)!vIl0b-qD*|c(DArt72!j-@vVw=5&_r~UPC|08yF{R?J%@6v2t=Jrc77M9uPt=6*wizto80M`TDxiUyAum0>ez zEz-_p9DHjW#`(&1Ul^YXcD|_s(K`KAp}=vQcc?eJXeI|-q9dKD7HJy{ z=M3Z#@KAw|I^gsH*~i%LM3- z90$sTjwO7wOlVrd-DLv#ZKkxACIaU-!w6Gk91`9shQaqOi3U*%Pcb#Y`3{?Vb+0n& z+MdbR?XSqsk+pQbLhzJ)qh}yVCrWbUZr19e-7iZW!JqWUAVC+Kg zvT>`YLJ%v1*BncUU2rY0a14q*Z{xR&$gKT=oL;c$EfjGO6ILM1u6g|)dZ?BEHm%sa z+68lHKUTKOj?)}1X1bB4@&>#;F5=`u3uZz~F;-d!(GS=nB$B5%%S6W)MFnj(IjaIT zF2XvD#`!Gb#tyfW6yz+YPhq5Hr7sflLv1=+uy9wnL{BM5aThGz`Qu^ksHuPYqYj zo>DIDE8`R^wq5r36FIth4d)Bh>7!=vXF@cPoeyGI!rPfP&Ax~ndRt|07xD|fbtg6& z;kwl=5Tx^3$cs`MQ>AO7l{~PgQzIc@unPxXKug7vhT4F>rzU2Ec?!*rzQg=vNwz$@ ziWwhO%v-8P5lV8?)3>^QO>y>&cKZZf>{-o%HKcTBnerk1KIP@*rx3%vx;tMqY_3%Ksu~ z{K+L$RQD@S9D55b@u@tyWY#Btw8dPd-em_>aD?OR3O{BHzhtgYpR#yiV(wUI)<0)d zfkDpX@-X|p$;GhxIpXtst?Fgj?e-k0evbK^pSMA5+p`qlmY=s_eXTLJjMGHSxVu$f zO3Z)WhLw?|vdy$YxD%Sx!W|`n+eT>=AfC&;xkOf<7F{S~*RDFT!iAXwZA26YunXMr z9sD}K|Jaddi)9!Ue}V@`K7L=WwCeQsI45}2b>6U%+=Uftm$sQ}0q_4`8B3=ekqEiq zI3H~2ZMlh&bL}mWlGc=S9Urjmy@#*x+DRn6;+04ToyEy-R-GpF^s7$SruF;~j~%4? zGd$pJg~wf5=0gNX5+DLBHg4j3M2W!qq@Ib~!=kQduf~bv_}`9JlY)gWRyp!AHYusg zeM{Xprw&~NCLfw?7uJyGK;YDPJB#1w2e-uu@Y;l{?dG} z&=$EAQ5TP457$Nq4n|2eve$Kk<9z1;=e8<1F5~2!8SkiO6x=&-t zki%?&7mJfActD57TaI%0eC8E1GE*6$QOtm34#qyf8PK+sXc(&mIBTlk`Pub|3Bf-R z<{e;$|CljjO00ROB+ ziGwct(IlcQ_jdEcS?5?KwbZ6kcJ8Ln6*{wuh-%Q2PySxgcaK=;C5D2gihIQ5HR(m= zJz|VH6V>W{YAoL8N~nMPb6|cLtoiT0OehY)wx2+y#}P<_lc8;PG$b)J!Ej_txBP4h zuc5}PBEe)2v+z*MEC9*nqLhuN-|(ntVNYbty_d*V2<#ujC$%{Ow0SwLM8aq;4?b|h-G>*i>1eOVZe1E9aI-M(M2;vJZ3lno)0ZA z;TPUH6$Um=w9;84OI6RZypl#bv{|3AhKF=4QcE)e8qFRshyB-`OCJk)M9}Ei_!g*% zvxoaWv-Kmpzb_?J|5l_e?fc=FM=J`?VS&RO=7|s@f^q^@aB_=vbh%6yvVM|GN{w~l*ZmCK&fH1ptL5PFT;dFQ7f>o~dWYIXY zt(s8@b?9tg>iVrKvx%MCZH1|1FqoHf1s-{X z9Slebl@QU_UBN~+T@zYmpCZ~_dzQi@i{F;>oNeS+#`@w3{YLYCbM)Bl+Jb9-EB%Q?A0FU zB%>5nU=JTKEDHm6w`II_N*3&c3E8ko*l0 z(l@l=#QtL#vIlu1eYaer59t>Nwe#;mwom1ztkAuXzoKBblZGF6v*3V{sa>%prtB2_ zWsr^F*smR7w3BgM^&4=V<~x;D?qE_n&O+BV^e6#-4I<7vIdLW#RXIl|{-GpW-NiqY zt=4KZM;61RIxAEa-kWsU?{pf^Uj0t3#lS{&+d4cj9L0gTt}wX~Niot0OX5sNgwvvCRN2A(n_x54xr%OZKARp#+WQFgcK8 zQV0ZoFbdwf$Wt3o`V)3OEMct*RpwCAi{%r9oNXwE4F6bceuRnvwSIZeHR~mbfgu&y zLKC`meo#v#5Qw*CD-l_!+RB8)+Q3ZsVNBMYDuUsm}@cYIek5whmOad?Il8bAxrq}B#A-x zfww#{knQ=W1qcp<%#AmbWX?sN0GRBlK!~h?vh+4}vzAOg=r&wAUiLa3WQkJdExEs1 z64g=IxC|d+nrTfxR{Vg?A$grcVNS_==;g=9-xR1=^w94A@j?)YGA}!GV?lkzkw2zh zszZN5o&2>N#}oIZpGL!jw& zy!FxVA`p#YCeh8h5Uh`R$(FskK%Z&2f_V?$1_g2ipejeXnvV3!ReDls0w7no2b1jP zJHQtyqGH0HO@s07V!FhPhfrzAjv`=<-mv{i;UDZ^NK%!R);pq>v!!l9rSmhSOVHzJ zZ|?bxHOxhAKn{ ztf8Ohx|Uh$w2y_Cm_B1}JWOP)^$Vz6_3Q77xUA%RHbtiX-L)POqfo4jr)e}5GYvJw zontk&lcTsaGjmoMIXH%&f9)84p7Y%5^hL)S!id^r3+!uLN=k&-!@89Nwt7e-B&b3y z`&P}qS2H$&dVd=F&Fz=AoG$QFb0z_a`2l2AYROy6sYQJ%c3cO(bWuRa)R1}7O1ZN` zKt8!jw%8jB9lPv+2&dS)3-V@mLSW_Md1cRO{I3#N*Af)o}mpg!zy zUhbLEjTQ#i?TZys_sKk7l#rT!hhvK438~_^Dic!vxT=bfI%a)%w8i~_cihJI1&7oMk$;*s$AFZKostX)CAWbMbTp< z%oI7w%Z$2IiKRi-mYmryxiE*C+^6(X-W0~0sZR!}FzOy~iwSes2)8umpPqea52!wl z{$zjfqHTo9-U6&Y!v`a=%2qgF(@J-yU3aGY#ciSfOy!m666%FT(|RSg`la)3M~I7% z&TrKT1>pa&OXn&-dSN;2gynFbkjxGt@IE0kl4hOQ9W4^HCNrXFu8yWAWtA-N_noud zVQNzAF7cDcvF=hoL#cH`_MzrsT}$Me9ASE5^PG`xn`s)0DGIBz9hO>qmR zCxnb5Xlh8p{$QfTM)}5Rpskr47eQ)1mBLN8L%@oZ6591HrKk%?mKGghcSPSo+x_65 z(H)r&{78FT9r#OUgU{K0Ip$6fd{%IK%Step1r;0f>68tA z5tzjfdE~J?#Fux}+2d$GMg14`FiZ}Kx}Su(t>?)YtN#6fo&*n0=}9ah`v>j)9zB=Y zHU;AfiGRJ{r6+r|CHBQN?2@y0iF+A9x8o2k)QvNdq$Tqsxt+`q_@yK4*4!eJy_O+j z-MJr@H9x(P>sG2(QLfKi1N1+yAD<<9644_WHc>_~U{Bs6i=kA8UWnu+Rig7sWyvD5 zMLX}6uy+##R2N@w*4K(5$(}ZU{1TR=wj?=|&8WqjeH84xL&aoh<4v(;A&7XGz}=k z+BH4;&Dr-!58ojoF>Rk~(K1S8kxh?}+BX(C)An<_IOsx_OB544PYe1+sR=7P#cIY} zP9In#UnQfWMg7hWN-O53XLg&t73!H9audp>s$Iunj0ecxC~>hBvKVDn8F z!eIssV0XE<&i!$2XiyS@jc|K#L&zAs2COn`k$O=E98U|L=dzMSng126VJ6`4T=r>mJ22Lp5kr!(=)OA|$P#*KEJ7ARX z<1&G_ALegKhORLzf<0qM-v6Cr5uD#)!zm6g3`;2BG$<5}C#qxJ&!|omNzWLFWW6*Q zo5^-dIQY$E>goDP(xs6_LpGeVyl)3~O|Y4anoN^1u(yZ0(w@pS%9CM_CXT2$q&Tfg zc)wroJgX$C^m_@E?;3U70ihdt?ZNhTWXAG274ELF=FL2kBhGv!=2W7v(KSl8a|ctI zd|{}x&>>PSr`fKxpxm_9qeFtOgOL zmuE#t&xNs+Q)N@t>h{8%Z!Xq@)s&(tsvlJKTsB{%DSEqEjSJV&Ttq%pR9KMzCcS!y zUUB+Nu`|vohU4vwv!R>Ejo-I34tOnIWg&f-5!;TkY>cmj_%MS$7}a&4pYD;=A%E10 z8evkDxhNOyGcOK2;*0L^+kc(z9^s|dOYFoW!XoE5@u(p?f%7HJTwuYAm+qdKd63R_U!TnMb0p`@gm=xP-{o4QVlEI4&=`l+=#2y5~GQw)*WR9qYkt#0bC zf@fG2%j0KGCpwiET<@X2`~cd!l3gQ&C&8GwTNGUg6=96IwZozsVr=&4AWxL0S;S<= zxzoifNce$Tx8w21wb$z5zD+dVIJRL{Wh9 zWOwLhhzw6z9b&Dgc5p4BIvAryk&V}>2*-kEDFa(S-4OWko#VE;{p9DK znl=>b!7wrdtmLV4!EKqRZh5rdHR5P8=zLJ!HECs)*Q|1osS}c_1#2ouUO{gM0z)~q z>~jhSXMcjfyHOKWzT!>E4*M_loPRj$_US>8ePoL0qa^#vjL)23#LbrUQpjTUgHyNi z+)?2&(@x6HS-CeO0VJ$m{H2f-b2 z9y)!O(*3&jKBW_OGFi!s^$|l8lQ4bJzWEL_(kW%&)*${x&=gT8DFGd_ zCs=97qw`E`xsXUZw==7Wuu1(sfs&ZZg#vTIydi4oO>$yXi)=Fs4XdQTrwyNrF0X(96oAM?QQPZ7na71Y9G|oQ|&RI z_CO`=Xq4@L6Ld~4*jyjD91Ngq{)`UA|EQ}AlL@e&=}98|M*IkjXauNUj@vnfNdm`l; zgPajmhqjVRrQeyFRxe_Hj9k#zXd1gP-O8X!1+>j=b+&h*(sZa^uF`a-x45bO>>ln+ z%qI4OnN}msf#o@29<&r96bd#7a}jFNvxha`gqOTijD>b#R8BG3`T3dTk!gE44ykl!g+)Q{q-D z2BeQi8b7@Z z(0l#ImwVgIZU@w5ZR(l3sHV8C4a~UC_8ngixCek-B^=D4+NoUzQ^N}c8c)}6ojz%d z%_$lZzmAHKN`=sMJ+qC?Ac~N6@8oppw!vBH#R*w1oi14e^i7+aW8qnH!^Tee-KJ-y z3sna=D(fgi(kJm$rVwokW>yW7d6SeO-mUNoas%-t%b}KssUe6{8yF{8QE?-xPFtut>v{fy{M5U~PGvf)XtR&n- zIf^K8=e6BsK!c5uw(RuV&Eh1cBRg!t!>sd$KQ**85(#T>-AE;YI0lEG#iFzRw$m~3_COJnq|7zXPBN}j5xOp>opUp=P@!>C6}{w@U}y?-f1kB&q*yM*$^>P2xu(D!b%IVML=a zVxzzm{D~a}Vb8_@vrMPgSI2;9izJ5h!{fH}YQ$iRTQ zvyTfDn&HIBzjTbl&ul_xFlNI}MtU10bk<3KrWhaYxO0oM&FWLn zYs$N;-H6(%l$4&{W88WZyL?qrh;^InoED_qHQCR8XhY74f@VSPt5}PiPU_q|3UcQx z>|)GilFH9xCTPA0t?!tkyY&spG%H8!)%@F+&A*XRh41{;|xR)%*{Gk&4 zXXxa@p!(FPaL8NXvcVQ)1S=jA8f4*%%h$(DcyVV@G$u_W-HIP%WopOgbmiFw4*p%~ zr(jegH;zf=KtDOYyQCr}7}PsKy;?&vQ=j7e@Kc8V^Bra%Uudn-(c2^;vnA!-d32^Y zB}n&G+vEgmx;1u~B$o^x_6McdTnj%_=ILUtPo4l~eS)TRkQ)Qb6`Mpuc0Pao#DwD@Ue7JT zjCBAP6C*iNC{dERao4#z9M5K`wo{DmFe}e{h8wWbGw6r|_rx@I2OEcBAS8lvDr`4v z>E<4piMdA*nFT3U6j*)%`}}h3&9^*3a_rd63y@Z7(s=iE*ysmL)Dt(`(<^W1&qen@*atgE$Yan(ckQ%IKHjBQTUge% zWA1L0%F5VeBr$SC9apqsoU!}re!X!4xuy|rq>mb6mUnikPt*eSD^#RYQkBl@wEduk z8a%f$gO*5qR4urW%#3@5US>iJc`j-6VXgv0_kO*0{TDM)K&UxVgV8a`MBVmp6p(4WH&OK%N+w46M#&U zFbuCnAy{HTOX8XVPYFRbX*}=i^E_WLN$jbvBwxq-5^e?a4|@*zim1uawwTD6$!?jN zh)uv5kvnepPHS%%AcA~5D8a66duF8WFD=pOK1(#n?k8Shn89L;(-A*-WU;}#-v+ND z1m0=*YRxSKVn({eOM#P58BXRYqs4XC%e?40z2sE%H`-2pPI-Y-nY!}>DWlNnz+E~@ zuYj{6!wlrUE#t`N(R}ayE=4@fsW_Z3;5=WN zZZ<}(Bj&DbYj;l5Go9mjp z3HpGJ;JtsRqQ30Sz25#YQdph*etSBQ_8iA6Zai zha*6%jv~%2hE9rz^!vli9fm0yJ9C>>v|XO&b7jCJp9uq%vRpB09#oDa;|(u z#s%k$GT*`-XJ?Ni%Vi7J{SI!(cpABeXO)aTHUQzKk=0tV zt4)i`E6jq@E)EW|N?0sTiNR#Ip}TmKX`Z}@1KD0y7cFdgIX7>%?<8UqoiY37O%A{o zn-OHg^S;Pz+mqCoq8c?Fvq;pF)8#ky+PaIgJ?~wd?em8MD6o`t`?YEN#$H>8k+BeYa(ePSQK%=$BXE*k!Rez=O++KI zT_Yn}S@pi_k?iiBa*erdiX1Tu8j5X-E*NAOtEc7;UmnT!Tkh@>9~YwC@54nL1!B07 zZ&jokJ7)(w$}N&erz4Q_hKKmc>CUqqbZD^Le^*y%0jCRV-OpY69&68-V%-$7Zqs(& ziE|st)>$uzaM)sjryN6?@4IVgv~cuRdht=$i*mo{j9xTTHjT>sq?*~iSo&CdF_N8U z(Aq9Rhx1~Wpb@f%)-FLE$<9^g?n!f^obZuWzEQBAK4~_vCzZeo?!}{ggd1%|AfqK} zM|0F#NVW&|B+WTTvI`8@Q|1JgeU}62XpY&@*}$Gs0xP&r1lWxR?2}7ipPU0b02QAi zN|1aQ$u2TLPn{FcQ=ZL%9#75zee!HTIqEMbWWl_6K;5ic`z}Yg0vL%_IH=?7J~(BB zzyyR;nVLS?ZnuupQ|t!$Sno%~2&33Sda?oLe*b=-o@~-QtS1a|$iLsKCz~y$>%~9^ z{QH!ioTbeA!7=?Du%CxaBkT)p+cypRN6H1?U zvl_`Hh4?7;HV^SJGl?`?kMrv=DA}Lc9o^0>t&v+u#gKF%jq30N=B%}u2!>o?2Pd z%AgbmXW^%%h)&#JmBShskTo!zTLa8M@WVUgV(S5hNt$ z+p_!Edp_ZMz6cri4|n$WyibO)<;iW;74ri;ENdH5;eGKEX1Sl7%ZVGiwHXvhDM_l5 zTlk%4(mPvns|O~zDRJD1>I45kAD)Oso&C2hW}-?5ImNu3s16##I6231Tycq!MIOmz zAUOw84j@JAc4^TkG?UYb_6OXCu*3FcHx%Q)=vxtbE3eccsFh<)YF)IP-qFh`_fqW> z8&!P)46D%C&o;f~Zb;tOJ9$@ch57AG!e$@Sx-;Kkhdd}GRbDP}D{3hi690qsL0=}M zNX0xq^TGZ_4Sh;n(9J^%47#o8nT69%Po~nToEE%8qo56xK~5WhS8S@W&Of~O38IN( zzjpj+3lW0(E`Yp`BqU1qXz{Gj<$OmOnN&qN`RH~8^c0gFAKb2$tQrZO{eGg)!t|Qx z3~0vI;oLnqP{A=m*T+a@pt#*Gbkfg5*Vg^1kX zsSvTjact3W-1)MCb1mw;4-TmxziwCD$Q*0kZjaV~`_}Dlw-sT(ZigV#*kH}(Vc_y_ z=Ca>^b(+Bjv8oyT*TbeaWZ%s!cffz4frAx|qE$#v>UDc-T3QR5b(0oQeD?WgyWxxn<-yXqm z0!XCqn`JaJcZ9M4=uv!7#9>}^HC%O~6LTru*vjrv(uG?`-7JTPZ_~%o(Kmu*%h>5@9!1=3B{6lbbHQZsN_@ z0BnRK6adsP`D%mI?1Ovd;!NqlAPFsU=yGPkAo;KaYA=brgWYuyvHoo54YKJL| z=mDtO>qGW=kmjS)jc{XmV%dpN%hHUXQgKAm7oDEi9%yYNtLB+wyE8hbl@Xn;I~pKr z^A%auEvvevs#|?kxBMgy=#%kNE=DEdC@d$uN&Z*#R+bAQ06Cf$IM z@3u3NbTetu)os@5?P1@J<<^WH%Ze8py#UAhdhB6%<)Ld`A-N-q*Q;M_!vKTJtmkvxpC z!=G!;u`S75;hQ}m#!!a)xQBwZ%T=U7S6)bm{$|3I)DnTk?mLddtDO99r<_Wp>&~oF z&=(}Zp(5EQZ#A@T__;VxYV_f^Yra-+?DNPUB^EgQyepaKXwL3D96BkTw1@D8~Z}tWVp&? zh?@k~FlZs5nw_Ib!I~Z07vG6sRY==jI_kqWv>4ASm*U#A0YY2bb*Ag`3N6~v&{b6C z2!0fleNGNb-L?2Qp={!=ob&de_=Bmg{yDGn8KpAVI;GgUu>eT_dEt<{PR$GHcQ~R192JmqhvRHW zFBGINC?LfnxnM}Yxo}A5z-f+jht(g?K)S>DPQ$qRf$d-P?w_l`>mx>dPQo(GYWocgoK34zSoQqegU zON#tjZkv@K$<7APB0tR+`KRZ~f06QUEb?QAh4PsQo-7&aRGH}g+5Nw;pdvadm9T#uIh8F{ zoNE`enEnEYQHFh$b?eH0m3o*#RK9^BH$#jRup@h4lRi-rcQ+d%SPzo`PIWvEI~utg z*aWRr$iZG;1ssSDusAR)L+vt7B1MhU>N&;p}rnllgK7`pV@%rEZWgeTgm}I^}tg0GAEz$X?6!WSsB? znop5DcE}M@OKxF8WStgo^{Fh>BLS+aKObUhAO4LXh-3bn`DczKpQYA6DsDVAF#@Fy zm+2h1Oohvp1uiW+9yXz#>BlScHp$gGC)g&T^4lNieeu1S(J~3a|>rGI>FaKFE zfWLxdk@$AX4O0LO3TGk}7;$C$@hODl*HWg@;ElDgk>g`^fe%BhXL_T5n^$ytHK(a! znLo01at3K@lavWNHcd%3#M$%5O>#A#hcM%i8SeFp5zJijC~ovB2-g>^ZR;)0An}`p z=~AqnqgD;WrNdWLba>@#WW`JtRi6jKkF&25pN*8dwn2Z=KcA+X9Xw1p z*k$bjyzWMC%mO{Zn;}Vn&5V(zJhE6_e;OzTuAfKm=Di(|Fr{i8U89PlEZt7?v`1>^ zucU@ux0#3VU6b6RmoV1MUiR^oZelERBM73x5_CA}lLMeuf_kMUM~v0r--YEJgD^$p zKXATu=PrR|!>rD@cxFfs1}!hjbUK#rp=kw7eJKZ7{DO*F5SX3!xr$UYa1wWB9iS$% z(s*PfXj)J7YEXxQx|snrEu>Aa z^K?XirC=3m7#gUF33XOh39lP(CsBb{oCzfWodL?fX9YD;Yorw^b(~R|T?<#X7J8ts z*P-h*<-0Ra6$Q03z0EIqNtl?ZH25McO6q*A_HCoka)BdtA(->8Qyz zyMH240?S<{u$)+H6vz|_l*rT(?6F9|W^SN1CjpQ-Cekm*KjNll^`VMQ#1i9dq^>*L z@)P_J5i+fiaw)?;YUbHl!6<`|F^JUyOIY>7RN{v#ufB?>S=Bqen>|7#*}=EV@;V@{ zgw0=$-$z1nCN#c#PFykM7pPBbqVuD584l5@BcGjs5)h!-e)}18-Y-wdlSK<-9LH(R#dVtH`>45?EZ1$as%F>PCC4;I;9Yd_TrqA(BU?b1)Z2 zP&@)>TVdz%a;1|_p!ob=Y~lw>7n8f0k*`M_Ng&wQcd6s#-67k8Q^>Q3z26YD<`2AI z1M9GFG2l(MGDlEA_Jx05D>b=)zQ%cNFkRuFryY}0^uG{PVhM3Yq-m8GUXi5TYWdM< zZk2;#3m>Mb)ZBQ8T{p&AUcd@wv0 z7{(%kzQkH$Mmlyb576FW^R{pb>~>PK4&Ad_UE@xn*%&lCpH`zpOljZz|7P!PfGxYK z`_6OD{eJhp_ilekdQ!LG+^YwE4c(}5+O4)E+kIZiARDEuxa2CON}kb7Q8lX9J&9T< zDrH8{PgVnBf>Sa@2C$h}7=eTdMqoLX!5$hL6ICqRa;#Kj7mh_jGD=w{vLR!+hM4*N z*V_Ah+;?Ah>(gMPhIj5cXP^DC_S)-vuZR|ooxa_k@D%K8^sgbP~X`vB-LpV1%Z)vw6E^x zweEOf%poq0FKRADz6`Ay>|We)<&wlj4~DO0`K06Xy*f4g?o3)Oz7U;EJpBuyBccj2 zqQs?Zf)NfwV??%K@$PGXh_XM$^n)hrseu;NSwuiDuALU%N=RgV?aUddp;vyHuvo;$ zRz=Cle<@AN7=h=$C}3GL3qq_%kC+ZED=mhPiw7F2Oj3pbu-em>EgpSNr??1B_$5@5 z_tc)r04L@RPKlN_R>IGvnNrjQ0i@pX*`*ARDc2sOr_yy!PON%uvh_M?6usBp<&UY$ z8GVJ~O;8) zdzr}B3tMA=e{&8`XEDXcH8S=y+LyV|RO*2qjBKwL2aA|}Vknh* zq_Jm=eN>KJDm9J(eCyIW$;>4XAb-p3yQv%yNM1Ghu3K#479;qY7}?Ukfb*gpHPy%t z<=s2CzvO9h7st@u4)my7e6o$w{Nc}UQJQC)I=@s~@T!bZTmTBE(78v-ho*rcUR|D$ zw79DoC24UONAWO$-NERlLn$@^j@AKVJC#lP>lvFxcZR{}{hjfB1jEBoDGXoe%gqcx zYm$hBU?^;-&xqhWrxu5!-M~J!L$Jd(?NG?t336Bf#p(nb zq|l1b=l>VXcEt3!&=EFwT1U>0P>3^r0j+E2;v+&jOl@D#6*_XdglIKIvF&EB)NXte zd*z^6ne9;>G#7*(tyG|KcwE?o$3u5A4LqCVjc-Cr zp{tW^+L<=K`E+fVuLxgW5Ls88&et=&|q;Pg?qPnipScapvQZw$ItNi)jYm0lWYac1#{Tb z`feD@6R3MWS9fRA?~>uCTRhv+*ZHesS$hB@U6oSZ{hV$$WU3wc@yytKdg;}*42xh zlVt@5b!Aukw76X4aq`w)%sa90lFtcS#pfQyE{o@a z8PP0rb*d}?2FtF~W5(sso?PTWnd}bt0hE&q!u^HeeW3Y-GCM&E`GY1a+R^J6Npk#s z-Xv1f)0ahDklq3hO(61v@JRX(GY| zUB!bE=e)p_Z~spK|AW>N2|QcvPPf+|EDUwpLsgoz?X?x%TwvU6mo4u$#I zu2 zmL4iq;NB79srT@Fdbop4{`Bxd$L;NU_#W%wxY0vlgs9A%nXR+_fWgzy!nQpGJxIrU zq4n^**u&S+!|x*F0PS3GW*GjQ_3(m54|i|h!@9T>dl*{Swuep)F0dX-jU!{58Q(xKAd9*YKyj$ExxggoJS4M`y6m-5?vx$I-pT`>eC)*8q93ae!?) zTc<(@*M_g51pt}9S`Npu-_FwoK>mMm1w>0-FjLCFQ*mqE+M~!Srxye5k}IVpexRoc zeUOs0L?*AX^*H(NJQ9w{-zllwx{_k+~?S{#dfg{~Zj zd?j;IMD~7{3mw{P8Tn~=q-|y`&KfFD6R@c~t8Q`c_PWKD+vpbWCww4i7LzM5XXK;2 zw+F$folo@R6+9uDY|muEkJsk=_DojSb$a=0_Gmv}ejxwuJ(E}YO3g3I|IQvxDYa#T zzI$j7`P|t|n)ml0XSUJ$__2p47wAV&6Y~r6@7XgM=6Sw$YVv*gW%-d)!2Wyl%TG<# zC`xjuKnAAtVS!p+thuNlxG9WxFg}Mk|+jY1XgkDBMPf!|I zNF=L_iiF~cXOJ{V2ar`)@wdA0$)`>6NprWuLQ>3{Ue8Ph`(J&&^6$nNxf1Hw! zfmF(67Jb615}-V5K^6wmpf&Tn;!51uK_ZiSyVjG~ZkeA+6{=n6S0Rg0uclMMN~PC(RKcri z3O=tZ`IQ?x9{7^_U~Twuu(cCZ0+5I5-Oze>fhxa*`kFw#u<9N8)spYy%RNxX*O62EqgL3E40@N)s{YlAQ%Kt z&=-)@lBlyS@Pjflta2)@uCF9hudZ0O?$H%9#JhFHE6j=__SVi$>E_>eSw_}hvPmlI zFV`H#peq_LnSN!;-N)pcq|!IGy@CsA@w zQr*+KUcrF~x?V{PoUX94_qRFlP`?*R^N<;%4iah)uh4yDgIWN1nA5t&yRpjMXFeto zT4F3PBY2F3S;+@MBy{(V9_2kqxx|QQ*AeO5V%jk&E{4WnE@`?>OlCGJpxr4Esq*E? zFyh75bMl7BE?e;7i!=D(lsTeA@t`3IPN(w8!}jFa>WRQ=Xv4HL&;~Rzw85I)KpUpZ zrVa2nrj2Q`X#-7c+CU$hHqhk}O@*xEx`JjlZA{aXX?>c2C@F9F>+GoHLeJVm=SA_M zkjsz`^Nb-O$Y)3i(ioD0B!;vgDY-@r`3;#tRzq&G5`Iw6K}thF;!ZBFc7911nkkH@6CzrmhR>G0XpdR_y ztR_5-vSg{cfUJs`0l9gh^=#<)U`zut{4$9(PC{JRW^mNbwbZYx@bE=U^LN*j@7Pl| z^X~fc9s7%C-rb!}(~1jNYBlPAH>9d8s0MG+W(WN^J~dvIO;Tfl@=N<^k!fIogxtlj zDD!M8JPYbfppY3VsN&7u9Si<#a6;a3R`8tekR71vj`!oUnKs&O)n^SqW#M?Fx97(S_~F`ciT46&PTlTfuO11%?_Y1v&B%;;H^`rf~qL(sLhy z^?y#2cT`Q**yHdIYas#mytp9&pAk*-84K;S!W6jxTg#a`1MO#=Mxf;#;q30-}5pC1Ux9Z zg}B}b2zZe3B6X5RtK4RR7!u~k#|z1f5uf6?Xa1MZ3kM+4O!4rC<*9DQ(9dfKYh2oJ zmH^)ELIR9`F&z?n2q+ODy&Do-3ki4-TJc7(i_tx#TrjQ)^w@_kb?Q_0w)|KcNo8wS zdD`YnPHE`*msHiL0=IfPSva@Gr6&q@v0;!OH#L#-(pM)#Q$BnU){oNSK|f6Bpceyg z{0)odtfy%NhjSYElG*=f+g2(Y4qZnxHSQHqK@O8U!Rr7sViF7jWl)4Ypl!LpB&(Tl zfskM_DMEmRi*uQbOjbnx zlpu62eR3QBESyM>I@!tpLrlYbI1x8taM~H?2=k})N#lAi~_Lf1*S4L%{kvtBOa$V6}bA=`_|(Ja*T^D;`&tK!~h zgSZExQL|ax6Q6&u`Ft}zM{%l`XCI6xJ5J!nbM@gKT=Dk?+r$U?eNXuP0KYfG@6-Ig zxpeay>1?P20STJwfF%WJ5U)tO@aUpG*Sx%j1(2>97BGOiXV;?T9GOZ7Ye4wPj_^Dn z8<^mPZI^H0493ez2&ySx$KvtxSlzPZxA-s8r$9Cz;P~6Qepo?Z)(qLQYBVKQv-mvx^jue@L#u zZZIe>YU-jzei7Zz(YoeJt-4w*4~sp5w4IkPPR@5%Rk@s#;t#Eb==52PB(-V<6t~6G zYY@-?5Dqr5Ig;H!&R#H0(_)_%FL2jWm>iMA|2Wg)j5U8zi|Pz5hMH4e*5X*H(Y?hXEneturOM!5eBD}HF6S*!jB{{@10WP5G~jzE=xGfV`JMvb1DV7fkT2pI*y~=Ol7wSQCM8ZBj_lI| z?52BUH*^+Us)vM|3^=K5>AJ+=^*5g4jFBWr@p8eofPl>SCTOw(#hxWlJlWOyUFlk zK`lnFWn6O-E9&xKN}ps*ciIV+D^A30M0K-w=6-C*1nUCR3=I=mpVU3JHqOxK>Wqyibk$*tx?*#COjk_4kLrpz`G~HVioT#L zRq`EIhjf+oD;1vwBYnX#vJs}!N72FMuV>SJm+bi)JWwJ`(Km2n zz**zAwKGCUG}D)>slf4ZK7u65ttAtx$xL{SDidmunVP}p7`*)RnzCueS*_J5U)pT# zXwFu?Ya(&KKprg18CEJ-uZA`G4pUVMgH ze=MZ5_>!r+_n7osK77g^x=36;{JK4Kc{n~4h8f0yIfgM{s9_8k*1#BSqxT}R8S!@J**px zb}ds)4*z*&f=x;+gzM|16TvIp*st>iDfm6dH7shQHiOhta-AdYF#y_sKD7X|E3(S;Af& zvxoPc6#E9Cfi?hjl_iJtysT5U{!B#QPQHbV!e*!GW=D-E@ziz&Ehm| z!&>Q5Nk{p9lw_qJ0+T!1nblF9{E2bNT@W7yRA{d4^-6+9*vK(EfpyW-?VyA3HqDj1 zr91vL5hN9>9{u0;$g(iB*5Ch&|M}4m@c#?1=d;$je%kGi60)*fblAHj8i%!PL%nN} z;hWPO*T0$Gbq5;`MdyG`7s0w}KH48NB3_8W%ZV0|6GXM0?q@hv?T_XHB6 z3FI~rqQ3GU6+-UY!CTr-ewa)9>bIc7Xo4vN^k&OkU6v_(hdOJ-ps|2Xns3qpzncTp zY)-p2FWS!76Y-)dOKQeMRB6pX42Hw*i!Hz08WgoV~nhAD@pLp2Ness zj<##J^3k`&Q9#<-nxhCjr4x7xTW5S(wLzr_RJ4)L3#3#ppt=!q3)CEvuWhYC=>Up0 zOi2@JiO&d9_t6}6R%dp0wAFZ{?KnN$aBDs|EVJrv|M)(lpD=3(t)g`^D!h8K-M@=@fdVcBpva=l3%w? z6oSdx!t2%+p1uOQO0Ci@JXK+rr57^0rh$wY%pfD+4|RwiiQz(pEHfRNnGG|RHgeY> zI<_*0n_1eeiF;`vIf>+RGNGzIw)5*i9H&7DQqm7BhaUY?deq#qrWuz@Qkl8Y7sTN7 zozrjfF^Y^*QuUDwfcnJTNWWKOLOL7;l7TEmR>hb?(z5DC*n||4)+?)}lV_3Erp^*W zYSWk*TO0aCB?S18C6t*?OR0%`h>zEj%~Tv;v#Ec%lESoYoeCiA)Ys^h;nM1?=Z%ZK z44l-)YOwQ6amEp%;38UfE&@=Ik|azZy($U+#UGoc9K(Qs?2H`qz(OU*C>37hC2=0( z_-u?e*Bzf&v4i@{`frmU#Ji~nI`X*5_U|LfbL zG^S~y6gJ}@!G0sj2FyX17vF-z8+v%gag0!JSX{IsEtj4sFt7ri=$krX0bdXC`={Yr zNc}A(PrsQd*}Lbt)6dX7@Bg;l6H~2sZ>>(8L-#lrr^<8N zk#FdnyXOhh&(OU&(6bpR;qPhEVMh_X9V0gp`m_B%GN=C>QdTAJ?#OM_D#&iG+0Y16 zQkeessm)DjL`b>Tk)x+o49{G~cv|nwWsG@lW>=(VY7BI270H>)VAOM6cx|s;83pOF&TD8e7`Jya8B`q zDjxHYcSJC&JTLXd!2LTRqAUFdGu(WG= z_`#3}!7`pMjvrmEGCa)(pTpCn8&(Rco-tBX1cVfEDHKz3;LOecQ~Q)vUWzomkSjO! zT)F2kF~agV248Z`X>20T%rxGc1_hYzgYn5ymZZWHzM#ehCqJcjvV_8_2n!7`miHS^2C69L764BA=NYo$ ziM-D?o~ewgS%K${O6JtS%W|ox2GKd|m40Of@{}5fz;A6l*QbHPmk>;iyj4w#x^4Zm zo(9W5r4i_v+Jc^K{a_|9w~U0w;6qkvxVtlg;eETZWrALVO;u>u^*3=Ag>OCK-b;`J z>mam=@tUgOOS!A;W5iq_PF6Zi*!yHbn03xI?>FwgM!qYzr%fORwDkiU#SbOR2>M6^m9?yEP44XPMsd_D!>w)C;dZ%9a2-m7m zR?32V4EW$QxYkhnV#m#4yOte6zn*DQbE@wSB=u$o#Ivv-b*|!)zqN*w`3)oX`uX-E zTqGv4`y)C;eWo#6(%uv+gvv1(mZPdy{<{X+Sd#HFa^*~&sV{U}b!C&~ZR)yT5ou+~ zP#4<4{JIp@q~RQzj4W`ftW=HAE3z_-Mb`am0@%l8Y63oSK!nsNmZU^L-w`H+fSdS= zxErCw39@#uq~@ayHj*aVtUB7{vFmIcctn8mJ|4si8G1N~^B%5?5Y(bYd}~J7kPjjj ziN0w&qBYJ>oNTOTu;h6OT0p6XKm3TjJ%u&T3nW>nSS;3~PxVA8MsD9CiP{|wNk2`FIIbd|S2SsWVY%@&| zTx`_?)t2%&OA1sY_{!tlv>BY33vG_hh(?PbAOKFuo>*l0Zrg^xgIK%s07HpB9-=mi zGc}amGth;&xQuc%g23=h<%S9Xjer8zvq1Egq)c zggU9HhC!`g5HFjRvR7;q=2?_=jXV<*Bz}xIQH2CKt7_~%DZB5%sau;G+1X~gd#4QTYN1nxZU!C1k&P@+I|YNWhEuA^ zy=)}Sclrwqe&}=}Gql_+Gshjo7zG3q3|jGQ>t8wJlxbpKlnFbx$wKX>y*ODYQNl6< zC1>NtWuDv_60IahSC;R+i$hf zLG&8UT6|SOQ9>_gOE>L0 zW|^UO0sLXqI=2u{L%gl)Q8Ut|bM z<88mJP|oRpX=Cw2b};S;`(yYQMw|u=F%+(bx&i(drt)JdjQ)O41Rwe+XGU7*&DB~^g z9DDa(1mM^pp9gR>$matb5Lv*HGT*ha8jLtJE8&Sd2uhK|sGX12XrSZ=yg6c=8LC2{ zYj<|eWhv0ix7=b7%cw~K7G?&@8o}IlNkB=qq_|hX4M+{dbt_&of!}d?~I7-sBE)pfY3^v9bD_&2M7PO!KsJy*J@*MhEfsN z0^@WltaDhvMrXWCkUte7R(;jbN)-IbZpB)+v2i!VBDib&m^_9$K%MXe#c5!G6Ky83 zljJRY*tO%*rwft0E;O?6x59bnstN8&{w#&Dp@MNTyY!v_{gNj8x)A}Xi=+&S5sJK4 z|4Zh%Xms`|+!I|^JVxp#_Y zW>P-iyX-8bN2hS|Dc%s(`_Hr$k1yH>l0yXe=?mcFW%ecC$4YDOOTGt1;tIV@Nej%F!W0f0i=j9C67s?;l7KqPl);W2u#;nW!8ObV4(W*lcUB zeZ!Q1wNgkb>rn_%p%cQXCWEe^`ejz5f3|R(TKl(Q(Nl$rhFnlM7q<`-dL}$nV3*n; zjU0aw;O$0MLJTG7D{jk(-)F^u_&GIUo-8{Rff4aKRrScuwbz*ckND_Gx$h$UE}x`sQD??pj}a$0_~X{6_Dt^%Z-%x%uq! zUF-dKoYIT_-I9-Z!0#3PrktLy@te>2b>!|-lx&kuYh4RSTPo5$cVXe_y&dtFhI&;) z{EQ9hXVp-z)=;n3P_Jx=s$xTZ8ls6)*V=L@A;6sZaBtP1)sa2mW%5197jI$KR&@F} zeLhk2Z@q<|{oCx0Un{rWa?6PyoY?9kq9m;Fbof*KYD0?Fek;6bp-$$^?D*ekk;s8O znzG)^lRWKgun$B(*}bXDZsDM0T{JgT2<&W3dUmv>MUqXKERljX{xjJ%c-F~>An1M; zOIn;%_8n2(-G_)t|!iw1#+DzFY+$^z2zDbg#_ zYq&sHuF~D*39D3eAsrQuC>f@pZN<7atL8z4YUF`wII7PdAc23z}8!8T1FUTJSUZfMb!!>LQMFwf+XeK76Tx<@ zcHk7zZ@}D_L}BAQ;r9Q%afUv2<8GvWeTsE3H1>YWKnK)=*}K3kjFd}d~uRYSEfW#flDr9wGh2o$VZ`RJ3D&ESd?2HN(8 z%Yf_u^l)7CwKhBZl4SkM|M|B+{>88UyNAA#$TA}Il_KRhtwn~_8w{9e?Qo_E?ynL~ zyer|Q|GM3PsRNu@gx7LD-h}O?24sm4_>~ zqP*`m)k(3aDQgvfVPt&K$@mZ27C)v3lq;5gh{bj4zRz!}*iuxvrb{|eXAsb#Hay?92n0vHy8Z?DME@m96|lHu2~>ey~L zP{$wB30-TFXKh%{!DR0#JFWMz(=1>MtuPM8N~yR7FcPGL^dJ_XeVCC4;3Q=1 zk#vfM$KSi5K0r=?SYhMeSBHmx+)ev&#hyhyaCXp_u}q7VeY(a1AbOfq@eOCUypbt2D&It~4ynRT_FPNNHdLE|WD+Y3O5TKD*K&XxCH*L{_8!7C~&klINrw*jc9b z-B?QA8Fd4h4}uhcZNs?$a{d;H78VD7nam>TA5vw#GZkXCclOYvYx`A6*LG{YvtKVw z@1Xyt_EgHw%x&UWt<}YdCC*d{ADYs*i5!Y6D>1$MD^eQh*sD0a|8~woY7U zdxxwGI%ijs&u--n#x8o?^+@<1eG0;eO^ylTT6#Q3Sf|5$b+aWPPK#T^^$x#6E61O- zL|clAw~B9K?ZF7GR`k-aq*G?IwJ#g_N%N+uWW(a2-++x3`-&%Zjrq2U|Mu&8`nux2 z`cn|A6mWi~RS7t<&6kRYbZ@E8EDtRMU=Z|+RqpoJHIV=xm1Lme6M#uY0=p1ch)Z*1 zB@GKNF-5lI3k}9VpUguZCI^((7`<&0y3!5{HZz!ja3oKUxzueH8b7%_dA&^>PeMd2 z1@}8$q#p4+JwV7&#ZwjyXvRJHoz5Q>yOtOkMTM9(RYK)z)zob~B`G=&eM{V+*AlSQSQloEh~9HIo!*Cg{G@^8M7 z%AAlh*jDExFkVz>3fjyZ&pRR=Z|0&Rkd7OeWJre$N05$G6T}+P1XWAiprU6t=?`_R zvF9ucv?B|&Bg0L^l$iJ#XotKs)SMR3j#NUlcMEaR=ihEco49{7tAxvq6Jozi9eXZB zM6eP<6QQ2>Os@;D@CMC7cy8mL89Np`$K)40_L(tLaed61p8L?(;h9B}rvTSsgKc;G zr8|9@f&C^wK9Eia`ue`K)aUl-{$HfyhlN`Dy&u(&%Nl$5ZQKt7<2d8wUF#H2eZ-d8 zjXb`Wrn-g`JSTJ3VQ^vr8e}7PDG)(|YFUnC^lHPBY!#9PE-XQCj+H2od$((<@t~nz z#&OjOXsnShQiH4KK9|Dupj0!K)v41gMpgTSUY?Gd4LN4%IWV5$o{s{8;xcOZ9O=R~ z?d}o%9siQFx@7pXor>Z~^zAss(-LtBrz1cxfbj=p#B~}4B%p!@>(i_>9SQ0G6!b@1 zyw7XzS4i=#I6&S&!=%DkU79?ZN4Vru+aAxi;uHMYJ^lk>gg#QXpq1m600CPvy}<_3 zowS6+(pE%#I)o{SaamtGvN->YI?MeRM8M=){0-Ag`o;yPS-4eEBnF|HO7b((TOq<# zC_O+yy&>q6iHA+QqUK%B9FtRYHA}(~K-iBz2HQ z#z(Z`xxZu%UuhgUVWTwzhXyAw$3B=bCH6rU%4kn1`w&XZ9u5)vfEUC`Waz`l`=EB3 z-2hybo%VAx2pZ(2fNyS!5n2gFT{8w^dnM1vag!C~ZK8H$tE13dK_2A9n<41~oC$6V z{2CIYD)CEqh+pf_f}C&Umj%bB-SNLSLPgyqRW+%m@@VKbpQd<96_-GTjMv0A38l3! zv9PKrIwa^V=pIN4h=FjO$7}9oT6NX(KyuVcJ{-hevong+A7TQMqBNPt4NdIoC|!p< zGeNErjB+C=y+S#~iX7sa9Z0~I;&UfACiz|yC6=FTlSRX(B0Z2?%tV8SzE@f}P}&5+ zM-iH~EC~EsQB-8!35me6X=gvvM(n5jh{`E}$k8)Dq9v{B1V5r4ViJ&g+H;dY-_-wG zKvreN@P;!z;}iEIW>(lt0t7FbNkD~LGlktG(65;URLcE`KB9=@pwPUOq2p4fk)G_KS^;r9G`q{4b3^!d14N{$kq)d?5Mke|U70dhwH+ zf6%}XpJqI5NfM-?C$*zJ{62v3UA8z3bZkR+3mZ_jIDC)bg9Zzz$X3#LP+5=`!S$Js zxRuN2Vl8ix-3|&PG~A}Ft*j<*WH}v3_U_UA02OIEymZU~fs)~8v&!lx{jawwDJY>d zui5ja!g$=JRcI-rQoC>zK?{<&03t-`p;wB!sCdP_IOJHuBq+X!SJQ%Ys<7R-%qu7*cTt0bx?Y+lm(u_VmcWAEnow8}H9+$et5s612U z%!j&;N)vYJ57rXwQ&0WgT|Ee*j%5)`8Wiu5ep-Yr03bur(Lu8{O3UuOp0H4oowHy7 zJCaCZgh=Y&hyjKz!>J9vS6rTtn6O_o=K|${IBS;==o7ct1M<{k8!Yodak+6o9^zDB zOCN*QX?9d02n{zBJ7M3Zic45p$`%M$$`=LKp@k|9(m{Bj17j9(cZ>ZIsssshNGuTq ziMAmJ3p0e|Bma?19^d>Xp(aEpAqU*X*?L2bB}OONjw$6Rm4s!YhDs|ui3^gQU31Ko z3(B%?9w{mgRah4YaT?GYNV-?zVj2)7S4Y<1S%77YLR?mzXYmWe3ji}Fuf15on+i}) zD7n1_iEv|P>EdlC>{J<&Izw0_b{2V&Nc>2Mw5kgP3x<0wLAF+=)hN@lGGiP;;GQI7 z!#&HmZk3O$a?SfZ#5RcAd@)r?ia9N@zAdm(HU9M38`aj1s= z0g-t^cp=?o2#^6y;b?!77Dw4&6~HGU)?H+ZI`e)@?$nB4oxW|76(?b4utP9o0|`PO zi(eVC*)YZEgt?+(EFF2exKNt`LJ>^u@;hE=@48ml1Q*goZa}St%ilEnu*B2q_WY?KtJMFDnK-v+R;y0ud>L!T6)R=&@x!} z)gm7@sKd~xgK(l=W*&zVzCuA1Cj*3lf5q+Tm~DKurfQ6G4ijV=Lg1ZD5Jl=(D9i^D z3-sze@}t|3Gy@{;aqXu*0|bL=2FkV141fsI6BmWbiWfl6kCR~X1$7JlrNCqJ7X%)& zXYing3#M3zNQZbYCff;e6cl)%Ri4&CEkOYrcLR*iAJmyZEsL}fYaJ8-Fcj;q$2OPS z3JZBED<`}&JL>=DOAZQpe+&!}=x0O+8vAUEH>}}Gc+fhPQve3&1_k6W+v;TePtvc4 zW+q6l(98YNLWA(OFI$xr+I}6B6*}QHWsX#3phHZ({3IJ6`fl+c?HL4wafih&~N=+D{w2C9!j)JyGd64HnEc!k)E@S^k_*oz#4MtD*B4(tWK z1Fxv`9oS=h2mZM99oS=h2mZM99oS<;`0$)x!FPaO**X@^?$%QEaJjx6J1&jZb z_W+I8=u?JBC}UeP?TX!zKtY-vc^%LRBU>YBfm9OnE2QO;)PqRtG>@u@DzTsNX$X$e zGE~63$S0+>1bcA_YH5mGQ(k$u@`BJM*al(zb$O{{)J|zy>t(u)GF>afTGJBrO{cUM zPRgKiwjoe?Ta~vPAGY;jWcb?Ss!hABu6|Q6w-6h0<|m~H-$3NE;gE${2sDL)Ljq{3 zb#V62r8!4p$GpqX3r>!Ispbe`idQUKm(3J+d*RzqP=Y7`MPQX82oOS%1kDc%ur`^Z z5}&zib2HI#Bt3nk$rJ&z!b6b}qN&4Va-+!{#>uovW-=+?n5gGFnX>zrlj&!7Sk-5l z%Di=H`u(6Pmn#)O>Hokviv_7>kmx#*QXSH?$0oWOkuif( zLxte|P15JN9@n3n%PC8Wl`B$cVe!8EJ}>7a^w8}LPJoiN^k3SGbb%K{d?lBrpV5OW zQXq^e2B?-aTwxDtfj}*_VF{GXILsM3A$An(kj5w!`E*1P74A=$!9k;Zo=35d6!Mk~~1EjE1RCbX0WaKM+ROU9>DN~(5lWnaEfxoOu+b||O z^x$R>aTS`?%oZFbl<(LxNpr?7;sp+RW()3&BRQ`LA*}?l3MD~v5M~hyRy-}p!5_=O zLj+GK%jL=UbHP@9C6}F-p;B0`3~?`LYIn5}YwrHOlmFkaLRD84rGaH0zo zAA-tM1)ilK(PaJo0_=AymSXNai)kA~E3`1ERJZcom0U2kWv$lQ5|Ht+)eJbp63qxi zh0OJeF9`xs6?o^aYQN%UgMq?ZQn+NfJ>#g`Y*{xN-;o%9brsT!;f&V13A4PtnprOU zDofhm)c7lK0|$o1kAP+hV+wKynx}$C*uMdP)kDRvbH}7YGCdS)zjOs5uU!;RVj*ZK zwDQUrZ=a1DIZ}%o!K0>`I4Zov{3SOXw^vfR1=D6nEfj_2BPvE8w}~FHEj744tdych z{K2$uRv&5{d)-K3R9woSk$CEA2tVyFW&Ko=IwTXZofUYb%2J)`y{jD@s|-eaoSX&=F7TaaY8B=CPy_LWD!vY%%HpK8-qkMEBoP4&*{yu z0<{4;PBs-9J4LmcBrFX!-T9JjyisV$90>f^s1eWvI-wHd;fwp8ay`D3svq z2NAk%$%nUKJ?!&;i_ODg-V7W3!F=I_u$=tV_gLRDQj!jqxZslZAl$iK%kth zMH`6#=FwL(t4J>*0E7@y*Lh@X2pvmq2yU;mKqEylETG0@>ng=1;~dXLI8YRG4=Q^CdK+%BCkJCd{?a5dQO-gHJf=hDI?`A=d{So z&9rbQl-ivSmblw}5=rq+3nLcBxc5OqM34kvY*NeW5Vr{s6!e&iZ&cT>NQv}OGuHz% zVG%>SPA@J`uH^#dyT;#N?eDJgJ76mV9$sM;!6j_@#H}Zdc)6QPx_6zXNB}`SyG|U+ z_NT6qSvzB=3?TCP`tds5`|$|X8_H^~cW!oFm%V86f9Pwyl`@Wc!0^39;l=*EUP&xMS;0 zjJCdn5NEZ7Zf3E>B}8uYu!b676;{rR)7^}++VIXn)~n^Its)?qbpmxX-Gt}p!cjoF z5CkaKH-C}l)22N`8vC^fX2F`l3QOqY*(@2VNtO(&x3Xl|W0XpNT-q`0F{TuMTv{>g zF;)zJTv{>gF;)zJTv{>gF*MpAmsSjWEGvdTudEmr_jny~Pl!rYEF@${>lDdKVg7o> zNP=1hBygvh2}9ySOX5m*ijs>dE^iw)I>LYe)2>|o;JIA=vMHFORa=HFY5?`z2k2SEmF)=N6L2P*$%7`JO@V^g2X>2roUq=IM8U@9*s9Ts)3&R)WkOu|zMHSSIvA{7F7ddt3BQ2q-<< zFl`bQcQ!niuec*$F?*sJS;qDr=BTT#CWt$>CFUUw3#H5A3CmSCW#)?yrnc^`r+{uz zc(e?!)U%ZBE{)OV@SXJN-(UA1Rq91?R)O>#DT9Ar# zBKSfpal7cp>#FESbu~mkUdwQ`pw&Iw1A|QfMe^*?k2Fb4rcn{!u~-E!uJ~hh3MiJE zS2K9=tO1Fi?o|Pan-u|~uUd*JB<9P^gW(ByKx0P>)BLoOxVswu#pNoG4ZUX44%Q>& ztzbFI#N~=3rAFF8;=tK#_48Af$`(}-1tFlTReqLF*I1w7sgo5eizSs6Z)KeZ*;5R! zB9p4Y36?z_d%U5Z5)l|%>yzywl6S2(8WA0 zZ=ie}M;qQi9ed>s6trL7U^<^Y85(km4y%1?>k&2K60a&`;W+p=OE9$ z$nG!$Xp$;dD}tR%#;$`V(?BrB|4B3GJYPe^jNkT=VaCTe?-Lr?1yHH1{<14-!mtJN z-n*E{-eHqZEhXOhyHWX0^;S43OC(=0?ZVdbjfCqN5UaHkV#Yhbh6r0@G2lVGH%c`RuJH-jvsMvg|QgBV+xpZb`K;U`v``V z%MwXe1hewsL)C-a9(s(LWC2k)vLz=m5mYB9(jM#Gr2 z>(-?Nei1E*umFBe`YnNw^@qG}r_uIlU8WaZFvCb{kc|(QCj-(Qtl7HR@SR%RC)`oP z?$x_)FPsf_k-_Niz(y^IO5zCfp!o^(KLf2-MTnC;A!dt?Mgsfger4Qa@7KUf`7Aq~ znf+TlnULtAwDU&?Yvsw2ivanh1+K0o4+&QfCkYR@`&76KSj5;vMHq$U-Oc#sK=K)t zQtT9V*-Mj$b$3M)+rju7zQOz4V#rFW-L~&O7r(idby6r#1)k_ z5+Bnk0k|wMnTa~|R;em{*h`e7Qiy8wSatujj(ubp%dfLcf+fDBm@HcQtoVzN@&9@J{XYf*pNcV8)x;B-2?gr@Mu@iS;%?VpH%aq z=Mm1}qr`DrNuwN6R!4*#Wie1)UMcI+<1;ao$&}eVw!rgI(2;6V-0l(SrZe)mnYOnavKp~`nz$61z=#L-6UhOAS|w@V`9k>|!%h6n^U7lYD#hpOdi+h99r1TgNG-60 zyCl95f?ud50MV(_a3>74c;cf#kB;1C3vt*C`B*fI}#%643dBu^wdh zplB7Ua&xo&Lwl5TE;D~|%NoV%^LI@+lO;mU{+xW2Ub(DT>J93XQwX&qFSfL}V)NuJ z@W8a_-e#^0-l<7+D$Zw<50v!eskn#)6Vfu?291}(V&BQ1WWi|)+OjWXhQNtr9^B%| z@UunnwQsI)~gyw1U!`B(+AX)6qvLbC%l zPqI-N8+Xfuc6Y?pEBz-K`(T~&Q^%cc1Bnc2tzz>Ifl4gSCWbC6f`RN%3=<-rS(veC zXY~EkZ$x$5!mF5TOLfdw;_2-TackCO2WVv`g?Obl(~ZOfG*K=)mL-)k3JWIZO+>Z1 zAJ;!ry;X5l;ZTjCnTeCR+0G6sSrR%Nt$#@u3(S})Lt|hz!()muZhlCKzCvyHb9+% zV?(Y|F4C+8Jy+CFEi7wLmmo+(AVS4rXq0J)A8b5P8Kn)RrZOza0yM1htjG%>B*|*K z_+{%~57a2`eSqc7>gdJf9IUxQk<92qWad#^^DqC-@4W9)&P|qSxm|#AT)S&YQvAhN z|LAKZ5zJ&|u0Jr#DDbM!J}uUZNWMZ7W0#uG5x{|WVwR-K@BsULL`EOJa~7=~8|7rA zHQZs~QoWo-<9UcE!yZY+7LS-Z z)+*?Qf5H=A|1DG$9`Pc^h3LyQi*n^T+LI}p`-|6&KPP+_qN&6x@yWo_Y~wlLU2r!M zpi-=Rk>)(I76g3$KTuFE-`x~yRas$1&MsJk8wR~Kk6jIKVlpsV6Sx`NO+0c** zgh+n%!dJ*b)I8+v*9rl%8`%GRSICHN)>nu{y;$YhnO8_Ln55P}iSMWd_iK$5A$<+9 zEoWOPK`zpcVjmZzFVy?a!s8I>>40bgy6{FI54dEZPVqx^qy&jLgt)zyzlh$)W{L9>^C@$AWLvcyf=$WA$(hont8h+3v(F}i1gZ>;9Ix(}2 zRz!0BWi7A~G@!enjnT!h_GITQ*Ph--aSTFOF|IE29P^UN7n0{7+2zS7H*ziFjjS4p zH}5W1KQI7>&LzcpdhkGMiaouhzx(LYWyzw#FaUIIsiBg;4n$c9X_kP?3fSacYC2e! zT#!@ESD;E;Qd!2hn5XC`{W91eK!%k_;ioV}X6g#-mLCyz7-J}OMfw!fc{0-7C(Qx8 zmV7)le{*tw%E>^jwdBK8u2yBYh%=iV&5mmNunZYH)$j3SY>%D5H4@S-)_^>dkPahJ zqzjKQxzNI@{06oc@k3v_MqH~sW{FtnFQjLF!Fxg?$!{*Fyn zv5F=RqYyTxIYicoN9Ro9rRmjIB#TV=vVi)plL%3yWkZsDctwge$+d+I$Hu`{6{ZA--NC!bht!YsqztRC8vbr_$ozWmvK7B`v5E2ZTLNz;& zVg@3QGLsF{7#DY8{e+grgrTKy-3U90#A8Pr>8B1dY!_}Q=lxbr3iR4`(Y_a-f_9j|uwFRW79kvft&>Zlvq zKVh3OI3wEIPUg1FiD*^ZN!2!56C<@_79?JPYlcPYMs6p~AQG|0aSdsyjsL4=SrjTK zGy_(=$eCbtJjDiP?p*k=->QV1)dmj3M>Iz;Qhn}tHMfYCwD?kyEzH-ckHH$p|K1f@ zhKMmuI-N=|$pa(~V5H9KkpTEP9CQk*14|PM@sk(qKJZ&4OHn&oH7cPaqdTWXqKFB`Qz)8_~#7Sie44gC*esd2#=t3Si=LZs%bBST12~4IF zkCp+MuF0|^na56AVpL91=y}1*h>K{4uz1q|rKXzhWC}*AhgySCtEQ_@#%-E;GW<;$ zftzR}vnLZQ*+V%IBsj=l54aV(M_r0+Vy1~%#GA!swKz%V8N7L?G-X9H{%@M~Wl@&& zK(eH|4kV-awbgadv^%~>q+qd860woa>q32+8&tTcEn139d*djpSJR%HuI_@&d%?vj zXCN+CWOshtD7}0jZ(<*Jp(fF`R#Vn}t1L%KnQv8jXysdNlyax)pypIHpKDjk$B!OS zED{HgumGvZYy_Adr0;Kz8A~Di1HaAw07jbdiq?c6qVb-~upJyy^9&YKHn{yTqFt-F z@rN!FjKJ37JthZc@b&h*B^hA)aRyQUL!CEZvZ#Ny1nBm&6}- z!#)4D|4(nv^j;N08|e#u9{%xSD?QnH+bA;bd#XTHVZH2gr@!($n~v{Avaj`^?xyYa z^bWD&eXWP>p1iJzErq=Vxqm^T@kIp#s(&!;T;`vb-^o1GY$D&LrqGbo_fVyuSC61+ z&Yjj{;%rk66=BtzeZha`w0U)bcYzLv)|tYtwuWPBAr}gz!GRm|*19D0&Fr!kdS~8~ z&zm>s#y_BPpp4yrm0l4L!!LP0J-J9i_OG$nbF` zZjW5YM1MB zZZ?8oOf0(S$6~(h6`3n^jO5M%k;;zRg$pg_;|;~HhhF1I8!u%&FfiXiF_w;sWAr{I4ZOSZs(qjW1gzjHX&v}37>#=gK9TrJbYnbjcI%H zk~YK*bbcj+$gxP%fEUIG!6%Ge7xco#f$eRkwzT{@)7L$HE!mU)+KCJ7v(?j#<}_FU z=5q>M!dVLli+50scWUw@i(L==)5MpBG)a;w?+|Ng)ZwQsSRL|^YG9iwC!I?xeCxEM zGYm*u6c0;K2BUCTf+aI54@>CO&M@fI&M@edXBbeG&M<%rQS;l=m!fLiP^99u;Su8H zIxt|U>%e+$Ie>sl7r zl5Z4?z!Kr{hRo-vZs-Y!Pt|RDDA`5Lu(E{Dw(jJ5u|12aR&Y=qI6-@Bu*6DrJirqeK^~sVl{Aw>hX<5vWNz(9!Gl*J~^}Z zkj_bagmYHgm=y{EsC$u@O{3!5MwxM=%-G71-&h)zTSR?3i&dRj)!W)bnQa0NY(v=kl01;{jh6Q~O&CC|dSFclbT=ffyAXgvVMLQeAD+wW4ukRQ4N zXe=;=Q)D=QMPzgytfWhqj{Vl~FE7eQReY!*3nvGCabL%46bkn8(6*)y3^m$?bp4-z*kqPKX?6*a_- z_)fWxQxmVe5Cg&&>pRjH8>?^8R^PeoT9;1B3RPAuICsdnLe_x?TtytE>quR1T?$C_DL|DR4}i8seymI)K3=D ztkv#lugC}vK=u*vWl^~`lqKS+5S3 z|LeC1GA&DAZWVAkGpQ~@rI*?x3?TMk5tE(5Deb_dk@7^eiY{&1SzU|b?51AG; zG=sdEvX=TrU1^9d1g}D2Aq|k27>L_F%@^sBy27m4s&hc+Lr$1o`mroaQ~o#mU*-?( zvi0@WCynoAs2LXjU74#);icmYZ_|9A1|WkI1A&>-G46npSaUl5b*-}!DEUyFSMV3- zwW!$mz11mRAoxwf8ndNK;lM1FcR}^57`bxdPRfoF2}}02MGd0LFs#rf6l^s6@zZ98 z<7YStUCMk6i?q^^V>&EzYK)8Y$F=y#(=P0&ZvIyWNb0d140?>H6XGTnJ~+NC2;8C`R{gI)y;H8Sgh!Xy0cyPj9MMM74SNEzhV#v3S-- zHsS&1M&+>dApjquGA$vS0^Mci6BO7F0vJ8;Kt&YqrMLlHwY8L>hWeqDNkY-2-PgSG z(NS%2wh9S+DJ%3A>AzZ}fc;X`O#EE>D)SQ(lEn5G=CogaBh5oIA0oM`)uBx{vX7TB z9YDawVbO^3XTnof?uuiid6Ip-z<~|NJo&G$Z$sGjWYGz`A)P{nJ@iP}MvpB{h!Y=T zoRA^ih$xU*q+1jiUPh#w4(FhqXf{Xz#eO-kZ}=jh9xkJ5wt;KBfh*Se=M^At7=N^_6?8thClXG(_YjEuJQ*vgIjS3zHkNqqf3B zumUErH-TAPWZT0OhMg|j_TW55SnQdK94nsEGB1X~2jC=2W)JrncsHe0fanZ}I47Oj zmiJKC)p#+25oQx|W<_M>Xkcm|ncO)faWZm-3NomvjM8wEH<=AjCOHb0W=G38v`8Xp zCzgi{d8b_qq_Kgkii*hzwXqA*zc?937^YPilN}IAmoIKmqr%}t=&?8|qM=CNq74lg z$H{nOFQ#M^#A-Hl{|q&UP<=E1(9#hK$kwMYv+>vNgl0Nij`|{(C>2Sn0teXdpuhoe z0s6<#x1qCa4yuz`1u{n}JDx{mk7`RzAhI|F1Ec7Y*$Uc<;G#YR#~-4D&$H;lpM(xe zT?|_j)`)W}5gUC~G@RY4Zi?D2}8MUs2Xvr zk*(Vw-5^UC|6ww@PME5HxzQ}@f|5Yf0N6yqUN}JXFOa~rwml6nThyH_ZN5DX)_>OR zX-fEoYm?5|_B1{@!}c^hsWXM=ZBLV_0W-aKRDXeoc?*FD&NdCg*qdIkdVgptqRQ|V z=!HKEp`LnB$kjqIbG0n!{emX_Iy5%91Utp!SR-|Oj~sbz^%(1(jf2=nAa zs||y^-l@o9EzDCk*D}o0RxD#+{Y~XX#4w+kdxxPNw5J50nT);F6bSkf4grS1 zNVPSHOO+M<|8nCX&tx3O;;+Q&7N9228FZJFhLZh-R@9V%JfNl_n)-n7pe%-}l5Ko2 zGCGYC0}2EGfgM48L*AQ^_Q^`>g%!noAUvl-n1$82I14K|QJh$yf zJg;e2JT-Zhr#0n@r^Xg}+SIK^KHH$WN%O6I=u$4JDm5SgaIn-<6og+<*aOLxS}WTr zGJI^*nrW{-0(`{Fc`|%k2w~+LIA2Pby~q$l-YDVJd3nI3)sh}7M*qP4SNuk1YaJDo zg%Xx#Of-@kmSQNp2WhboHb^a^nL&Cl!7~P#Za>J5 zL_BAy!?~x#{l`{Io8vp$Y5-Js+m=t|m=T?8$-QTx44epYY+`npy|QW{R6xzOH&c#D zu_ivpzz;=xeUjY0p5Aq>Mk!)WGI4gDh#+F7#ApVD5#Rm)Z`zZ0_uz`@;Hl`mX%8Mb zNW469%XRjQ$3>Bzm>`jh_e^%?i<5z2!A?JuF|-14moKHOc18I2Y6^MS=i(=>nlBzv zWFVkH#eW2!ri_-;T^gzO5)7mO(TQEUG7qB4J@VJs5b~jhpa3cK5SR&G364$2T{+m8 z?6QyP8+a(6@J`jUD^>#AI;K+4P&+AgAS)q_77tk*L5R;`fi0^yz&#%TnpW|J#%mqu zabM#}s`0UtM|MLt?bxTL5z%eJ4~Ww<4miBCli~z9Dknwa@+807H6rEjg1LuxGv*$i z+)#dZSZ{1Vnc8JjC( z6DyUGYC4GrGChzgZuPFp@?2Z*KE(UxcISb5cf8<(a|=FHFQ|T_5@r6u-RZgaoG;XO z%|;m8?C~O2!fbNvG7Szeg6~q|PX&l9(!apmFq^eEa(5oj-Bv z?tAV%y?OGk;4R}EERG?Dakh%nZ?j`yM23ju<-=*0Pf-D?k8omFcYjYSy8htpe8(r^ z`Yq^7=YPAuXx(&Ixtg165WL@K1~uobY> z*%2r69iOuNHi`*|8TAv$CVa1}i(r&#RH?#c<}E zlmLH0^rf3G%%LyRk5^?~9HPoC1m|=+k{i+^LvCQ!8pY|*!ElBbrZK2WxS`K%04#3$UyUw7N%mzcnklpTs-~dy*dsI^ zd*m9A*aUPcZ{J1Bo;%QF#U!SR_8>)9jA75nbqGxAUp#bp_@1mI!kyaxotnHTr2DNe z66uDXLx_9ytWLq#(004U^+v&6`Y&!U!i=*t+RH%KyJiXVS|H4?pCQcGHwg1Tdud>D zFFTcxLSHyLn-3FN03x#rFmYyeW|)W}94~fl6t|yXyM>xn+}bf zk#PBbq;FBGDgVPh!)8ia3*;G4BDWtO$lc*q@1@iL{2e*0apva5GeWrg&zpxI*w?booX+^hX4x^ zLE;-9gh&yhHix`f9BAH!1};q&Ou8OM+NL4XXVJ!~QO1gh-34%}JCmU558h4^O*#*C zL2DtxEgu?1Q_R?oW5r|X@s06&McD73?mRIaXh&PEw!5agtOxLj-L0eOVw`8nA9ZH8 z6so*G_nT{jtl!Red1I(zRuUtw$Jl8N)!VUR;uJ#>3s=^NGQIG&=}tuyEW;dbR~;L< z?z5wGTw)nrQ0%xyfp)ti{^o(l>FvsgccB#%ii#V>2rHAPP(gymWhEfi0%si*)l!E6c5@Z5P z-l#O|isREXc!ZE=i)ncV_zcLhP$*@jh3Q1B?%ll}W#sfOpz_VX`tSeWU;NiU{@gc{ zQv_Kua$&-A`9Kj>>Xee44Z{3bpoKL}KrS*t7aQyVs?L9Bf?Tf zRGwo+S_%a!@|D^`=f)PU=n;0un1ldKF)8pa&L4|qK_&$0+03e@SIfo`mH9MU)E6-T zXBP~x$P`;}*H;g?ORf$1Qd!EtfIq@tY!=*um7}fSMf~N2B*$Me16s)vf0e;%EB?kK zNvU|@n8@F;xA2`qiq=@V3|_sdp|(*49mV4KEzlXtbK(!yH1?jo4Oxee`9UzdqRg?A zQPUbVN$_ABCKJk?f{rL>7uj@2DzQ^t(cL(dAVymyjF6NiWG0_2F_2YM_0o#G4(ffV zLm54&qYp++`|Qb8yarU6j%&fPDbWO#1o|xMc>BhQ4UJ0AWmXXmFgBH1u$ipN_+#c< z-ad^vuI=Tpq9t(J3iH_Swh>dfM&4uZD)oqw*y#oVM-*qy8g>E=4HZvO1tZCLDc>~~ z|4`Li_PZK0P@q0Mp>Iwb3}LlbZGXjjmnBPkVz>l5c+6EY{J>JT;<@SDE|IMOdu4;` zWy9GR?P#4X6t@#!02+{;2Qg!Y2kS-P!6D&;5bRr+2jBWF#DbCf zULp?c%=A34>v=HZ^PF|chY#09 zfalJK=kVYHx>aMo%-dwAeC}{wGGxgoDK*yw#K?C|2p>TM%(x~W>8#9H@?AoXl{5}E zz`iK{fDt%$#TMEp5a73b7x^yfu$=FLQC70#yR)*d%Z=t0giDI&4&b@%z;kH{9wwfv z;OCLJ6JF}Tog|*YbKB0H>^bGdas&k!t1;j#-zgs0&UfY&`)eBuT}#?LhnMr?aH0}^ zSxfq{eE)EQ-l6Bjc$j2}jOZ-TyHjVD(taAu5^+*&H84v&Qt}CuWg-vNvfEj$HTo0Q z+CY5CSfjo&)+qb*3t)}&nV-bo?YZU|SmU36o3cjCYeUmVjWvF|^TM0X#tV16OzP;! z?E2sKyl@5HH;~PNZI+@Yy5|%~Q9I>Mw2%}9p~d@r6n>$;!Zn|pmb!Ahc%?~X-{Flf zHE(1d8@!Qw*@OR<@kT-6#pjK%R^?VOUW#rxa$d@1Eg45%IzF68bB334Sxa`u^1BZw zyS@DGmyXA>oOf1h80E%!ZdkeBMm+aD<8M!ObZs8bl}$o6;4|uz}U-bHsI_3I(RPI2ez1i2ge9JsOz8YEmC>I0^F7b zm_HiLZpraEn!ucIOp>qk&wexnvpds1CpERMRRvCiZ652<>5Ud`E z$I6@3RuC9+e!L%WAJ%7E-n;>S%6rzccyxKsdKP1j_vE22fJyv1bwQs}G9dxIx)k5BP$8&4J#DxNGgHS$I?3Y%U^CJxtEO^-E2)p#jP*Y#C+ z7(b+_T6r$cldZh*WGjz3*?F>+H=bDKGD570jzuD;df}O+EQ+$oBEVx3xqjve5 z&3OIB?5+HY%I}TkjoDlIy(*u_^2Y3~e6I4VvAi*RE5EApYq9*=;bhIn%VjOuAItAQ zob0CujF-z=axj)ZcsM!e<+-dShhq6dhm%8Ip37QtB$hvNI630wxvV8u#`0GlPOkLw zT-K7KvHa1)i9Btz%VjOODwegb6HDnjOA}U zoZRT;xvV8`j^*EcIC-;|=dzaE6wBXqIJwEo-{eA=&NWc7&I|=j8e6;cwJgYyQ93vD zTuy_tM*>7bzM~}C-^%`s|MG3>&)88Hr2eM4`v2kC1ge+Oy|Dv%>Ze|kV7toH@>09{ z8AHRpp?kSE@Um*T%ZkG4FHI@9UFVrn&M^pY$8i*{??_p*(SI+6DTR4#m{Pbe{$W7Z zzdf#hL7_6GU~+g-T>qFf=DRlzME3?iE_e5aE-zWAKrmT^wfb%B#&~v#bDTmf-Mt)L z)b_12ScBPE+V{&S#-qop!F5U=75=;ZEd^UW2u9vhymezKOHxAr+670oJ*Irp_gnb_=qH zz(z>)D1p+CyDm(QBT;7ilpj2HB^V&+JUcWB&a<<~Ug}z%TK4+R_|Lb40TmtE7@tz( zNOI&OvQ2XKl$%gBuo?fXieyLCLp%G_`l_=}ySK(DYNAK{z$@?EEL0h*Ol=LY>e+m_ z-Xl1Y&&lWT)TscSBNJeuP8_nRaeiE>?YEfobrN6rg7iQ?p$|%YQTYf4?%_7p|0TTXv_Uv}uFnSMrY(;pCAX`vdp%TR?{vy7#Pvy`GTZD&5NcdF zC&52$Yf=JaYFgbG|A7!hVY8s@mbEjteKKWeLmcr?uz0~S}+d2-&5u%3G3|3eTavu^{Y$S{$VD#Lcl^9(ElH={RAS%{2 z#vjorRL;glf}j^PF5;4HH2jU|=+iohbm{HBq6D`(Zf+%AXCCBpShnXld}ScFUVqle z0#tGf`YFFk>Rr z3XtgxD^d8Yc-GIZVtb1}gv495yv0qdZ7;S@<*AgD0-)zx|0zGaj(z^$*^)}H+i%)~ zo$SO?j|t9_tR=7?NcIZ^tWzPx#^o4^vdG~dN%<}QGk(k#HT~XOW})VWGqRMB#*35o z4$h~1Rf*@TijTuJWWSt*RD>LHCak!9vKX#Q-u>2=Vay){eO>CNNs_eNw)f1T{Z1`4 z9l-Kr3YKyGGG}7YAsr^FMCNU_(pWxo(l&<)s(-?G{S(tDX5k401-${QgRH*2{=nCW zym?^KZX6X{#(sD}&O8@wZ`WHQS;qKF+7ZlQt#nPuz!9eU@l?vb6t3XJE=8!a73*1g zqLk-f*Oe1u5r>GpwQ~ST_Hmd`xCdjNwt;u;KQjf5I z^bMQ~Tpy~%BabKNzgR;>p|_!)|NMsJa9f_fS_TP9y6JJ!cSEwApXwP5)HL+>dSdX-JNf!?Pey>5cufZQsx z2oXk(Q?s#*6Ydcq!8v!OB-3WUA{ZwBQe(7sH=UjSWMP;TJ%-`aPaeZ2Z4)Fgy{4`x869J4jJ6@9!FRClz+_H8;m=^gW$zf3n6q3gg{=^5fkEyA|_> z!PU$=(z#&Hyc}mUJA)HNDI7G2VDLqQ!PW`}puPzMyPM9Q{$#;`Q^^hIPyf!rAPlaC z!C3U{FnBsJ*g>vcU{EKYE5BwiSgK&)`6VhLZrQKm(U2!5#5rlkgg9s1)LMyojllrn zZ9*L3Z9*LFbmBt%ZBewfgm@TS4TA~9A7HQ(7(9br>cAja4x)oc4F*rEU_eTUCLNIa zONwlI5_GU3447Per(qBVSHoaQ$LTs9JQEl^3%3g}Sgw`%JKis4zHKuY6b=J3vrbT% z&jKBAfM!hxMSuYZYZ@KQ0v$jXPmB&`42yB1%){V-K`}!PRy0s(-sGg2%lqU8!DpS8 z$~8g|On2&p%lkc2bR(%`C9#e z*MA!B%4e`|iMDFN5ZZe7?CIsRtG0G%TD5~&UYR^Mx#fej^&;BZLBN*Ux=|0*_zpR3 zcyL4gz-XudCs=zh*sLSoYsZD%R^STqH2{#6H9}_jKog2 z5wfV`+Rzdcj%Hdny*g4gbmNoI&<&e36rs(Ohc<7?X0X1lZVPQLpVZ)rDI4y$?63?V zRCaS{e0gD%TM1aE6u~JG=i2-J4dX~>(8zz~ZLE(QnKYS_^b2d*9@i{rTTNz)9p(eh zlnMw3a8_W69MbO@Vg7@KV)G{Q4IN7N(6xd&txaf)(}|kICcKm??$9SsBb%X&Xzp+4 zo^7ActklU|4c(}xo#I)AZjN}7CA+~xt@1l76+C|?MXvYq&nR?rofnzd&9&SV^Mb>z zcoQY#17Z@a)uT+?>E&?xJ8_Z*X*$Htfve22-Nh3biC2k;P%kj)gI zf@LY9p1`dj9Cqjwh1j96-CN6IoEa51@~}}$t-iT=rpi34S--Uy>u<1IsQ;O&|Cv_5 zWjKEp^|kA-O(g2ysrsMMtlwIU^*?i6|97hXomRhXa(D*#Zr2|Ka^p;e9SG?P4na3G zUY~)!h_C4q&~7Ltx@XxEdCF9#kt;Ev*_`A<~iA$0N2iQvdgI_IVXEfk?oUWT_Ydf#awqtCeP2a_hhed z3335fpXqf(_4&-EhA|F|>T_o#VCBU- zruxLzZStLH0`(ahit6);G!)h61`VMzLqkz#o?Jufv@6OSJ<}Y|v1}lOzv;*|X%qMc z4$oxVPVW;xs!nqv^LtdbZCA%UxgK4Kiebf;Sy`j<=as@>ClHIS)pK0D)&jO+>v{A~ zh$usOaVh?0xAt&AUPYUtCvmVVB&gL_VS^o1xH^*&Pgi$g$8+$Q7DzjV`UR2u0_nQ7 z5OavLy5L>2zPM*Ra)N;39HQq94uPU?gQAbBPgtTy0szAjwpD3{Xsu#nD^FA+s-AnR zwSo8MPaq#EtVFKTkj0cMtw%Pr2TyDfFHxzL#yj2rbM8?I6-jcGI6e<^jsllWa*tk? zW_7mdXs-g;dL2pD1K)Rmb7M`|Q8e*{rm5!F3CaC@p)B(?xP*?xC8PvC3QI`ixA!59 z?-CUzCX6h$GZ82ba1u+Sw3B>KhFHDi=khg%@j^BvShgNyK9lM0(s1L_9V*q{j_8 z#A6$l9=FBNMSGz8Xdz1m6|`w(i`8khIuOplvqPb@NF&omb#2V)9HPy8VCkifVft9o zQ7SU1n7QSvI|c(}TV|4Jg#$21fXaa+bEpw^)*oD0XLB~~Cnd&(0=*yDgtJs$Xzm~> zsNa-Ji2RWZn z4Nq-l)m(KZFx2&RjFM7RG$clWX~ZbxXWo!eoK4(&qsatR1E)+sY13jk1oe&3|s+#(6+JWw(8*~vDk3_7^|aV3s8Zk^15o?v=&p;67*$!VN4 zgme%w7os_>XIIoGOQ}uLKe?47g0q-iuc=^AQmo`vypYk-2$1Lgm z>F-N0a;TujWWI)TGWkHP!~7D|Qdz)sRQ-i$WdF*}W9nu~NBwFj9Uhy~(c^~F;jt+l zJ#Hu+9-AW3C|dsMT?a5 zc_lkb`Ji19>{bd%Fl`YF{j-y#e9lXiKD=M#Y{-h>w9;g--9OJu9ZSo%z2{Q)wudZ} zC=Xc`!A(FG)!|tQsV5b=FdtiL(2>Rc=y+tE;Wbl2%x7JyM7>=6kdmM;VdE5&%VraG zexe)7b|}juEA@IEv%--m<>c~(QeBne#VMC#Mm?113rzzi<@c{H=c8nl=q@jF%?8Hy zFilcC!^)`noR^Y&R9Vrj$Q6Xm?FJtDtTKlZcdwJ(f9TS3q;v(Z z(w`TQUwK$=A*r&QEf&1Iuk+Soc7dT_q=zQDl=U zb;gPtnT~}o{u96gXj;TU3ue#y0l^qF0C+$?(9tp#eDXHP!JFWtbRnFEQ6bR3B=LR& zvI}35PPNO24%RyXtBiVxo5RPRTEsm}2nSju?kvd`iF-4!Cg0ZrXVwB|x(H4qH-VFE z{|n$Gp?CwF3ts}x4&aoV%QaQcrNcoTplm|I^SZ+Dc7R|I4nqbOgu^fh2OZH@2?swp zuq7PGmy2*{mGUD4hYjJt%J#^lKrhiL=^P0(z1xI;&>cm-oDC8hqCcE$VnzFDKRlS> ze$)w#h&Bli4q`$+qD=deicxMrvy4`n0g1l6FWEQRmrU<|I~qn4q3zoK$`J`vFJkPM z!7r$QKH@ZR}A$RI&XplKu zE+UgLhm+DPl=($y5)SmcggX=te|N2xhvc=N{kv?^rtrz%WYKIKmu~7+O>wTN zW(_j0?QayjL_dXw_kub4Q&8kYc)cdMo_Y>=pRNx6!DGLSl*x?8D2$UO^ z_Ky$JUbM7tYt4F(uD3=miL>6<%+jJ|!q?U|2h_TzexzZW=pkLkZ{+H`VVk^mPkupD zhsNtotnPlAC3&8Aa$}ZkuZK&T3F^)T50}ITi$xyhN;-zS@W%N|bonp4mzNV*eR6k6 z+>oHCajKyXw-UU&eB`5wHkA#-#^VEv&qbIQIN8X_m3o`O$z`kJ!O6wSn@|yhC|>I} z(@u^C@z(k2Pk&dWrnONaz40NorTE(1?cJt7*UlM={#@49;`+}zf36)t#@vrnoU1dP zY}HY_sUgw}S@5-O_ZsouN_L4m6vQhmU9UvEHf%c{n{6lfHEcWIk>XU)u3D;a=);^f@Qh^cE z4G5f>bQawEA}xsrgvD$5iNrm<#pXURuXR_a(FxD|5NSCuqEf7&Ve@`WxN zJ~JCMrtpKt6khKm2{6YOK*Ho!Fr)De*a(-ul-hh)8Wir@UFuXaeox-lK`lD@ zhXdd0Q#huGa4$k*LRZLZfY$&Mnki!Erb`i@oqcCqz3;K)%KB`8E1+tI*_lD~jGZj# z^2*Id$jHFM`CiDG19y}PVa|!%hv!QiFiaeknVv{|}xzNAA2Wd;?Ked$|Ut?)(%I*$SG ze5}fuI;d!chyjR@+~WssH(c7$cqthgQ6aEzEa1X{;Xl^O&o@z`4gaj%Ax z_KJ=I$c+$K3ZQyti|EL~NZ>(0(>t?XdPN=(l?Gp_EsNS$X~0fa;3GwdkRurxBY&x! zUto$>Ho#QRkSQ@FOlf*MFgOz!Z^+^B!o%}S$jLTEP8N})HB*{#^uS$0kJ4_Bs9Tuf zR7CoN)5*{xmzv^&g_!Khc@_{eEQT8(2IeGK0K@=K5R(WoWitN5ypvwhGpfx*Q;fF> z65H5JM*IL0_FW$bNHXe4W$Umm+ z%tl%)r;+2(eaF#zS$@K7gXIEdGCD**aD7!2Qz2iblc}YY&vs@75zex_`#eYE4CD>9iILskl zkU(cs66iz{7||aXza_)OB+KOi$uLpD5EQO-qJmS z3WA+ANqCBV6&y`=Q~6kO>D;`rJW#Tjt|yDJM~UG@)6+|Jx(!$tvqCh|mIBEZ6Bfv= zx#S|^)jZoee!CK>$vhEj_%;{W?n5CsT0L1APq-+0TQoL_dC3fpfUU*(E!~Z2>JYYY zr=jm8HUm)!MnRw}mO;$LMxAteNU9%jh^P<{2w9T$AY|vmMWPzHLPXk0%vv}w69&4t z#DCRkxF*871*@n?AI_UOJnxC9dYckeFA~*+{@`nJoYTWTl2xxdoaT+1Y&{JL>gpE&|?GhnNyLd1qoyu1fuj4gsM~TuZ%6^DdoApoh>S zjhiPDIkAr*+-wBok?FpgT(#MH)d|k1m7O6vjerTRA&~tf=(w9Vq1K}(POXo+2z-iC zs}59xTJN}`Nv*fr-T!#hy5`$ckw;L%T3E|@DJf=;RDYmYzH+|2q0$cfEfumT#Z6-z=hkRqhG1N-)wULd z#-MgH)#g^;lQZaSufgiLhh3X~OkL|y6Jjn)UC_XUX`3u3IQsl?Mh{eahTXyO7&C)t z%;Y9+U5VJyCp>7_F7^l^FD0_M?K0S3dU6`}e=}9qd>NvUN-`tHChLg>|0mc?cBM&p zoZJ#fIvFr-u2criv1}1D;IKjb2!}aqD>8v~rn#d2I7UE*C*DR288m;4HRj8{W-ObF z$*pDJiWzOjITQR)2b3rC0{yQD7uhIP$1oi$gWDa($FN$Wk-fRma&O*~qe5r8n#01x z{PCXGZmb5pCH~znAxDyoxXO`KWHVeNv(zM1b+mh09{FJfoHftFRf?nL+3i<8{?YDy zLX-?*+s{o!qKp6>wx!ACU{+6nuRydcu}yNO?`5f6aR)epbZTPtiUX!tvy``wq+>q8 zyO3ocEI-=`!c2=5R*cYp$K+~0mD#_w*t+QcaD~8{icP96Mdxf%@P}_A=l^PHr)Yyx z=2mk#AyI)^xFCnG2_)nW5qTU9IH5j>%AQUvap@y2Jvd4FN=Rg^a$|YR#{XCtJ@kfu z0F0XHIdQSHF-9B6Ze!F?y#Yq=O`if7)$yc%vl&L;BX{EBA>EuuTPFDAT!}INeWxvc z;WZLPcgSN8S(shM${f>O0wOl>DKh1o4P}xUe|Cz#&1cCUdw8D9+R8Vzw(@9g;Y5e! za;N8u{@&iSFC&id&W8!m`vs8&e`sFnW9*MOBWd^JTbjIXZjMnqHkIVlOB@z5N*LKR{*;NmkI+Q9Xa?0Kw?#}y!}d+%G5hK&{9DQP_x*Zme*zqm)Ay75FH1&XDe&Sjp z{*(&{#t*1l_<|R)E<>bE!M8fa5)rNtvlvnvpPCufAY55WzEIr@@yS?nj&(;kB<*kT zIkPfi^x&AU-fTB_qrIup@L5kAY(OGt7}$YakcRDbd-7KSny}kX(rqg;i6kjKJP7Lf z1hSSvY4itM;&FySM0TckMitL??<2TzGdm-ue{(u5;P!RFgVO zWM!GoPAisYPs{ffDeEO1LVZ4&vtYSva}SH7!ptPt#3UFsuzWf@9TC-4P*xwrW37qS zGoc*e)HkFL_+=#mr0K_{SHGAb1TmFP+k*rj6y#?~Ku%wjIp0 z<5_uqu7!v`Pi%n}5vR*=)a#B|gR3wQ)I*)6)!$zX z&tDn>6{yitG7Uk#^9E9Zeu_GwriwpQ!2~ft@GJBc=+{w7Z;S-Uw1EbOz>L#%aUo$^ zMT+%dV#T*|0KUycc8wr@a!Mtq&BbrzzV5n>V{B&W}Hc#$w^i%gvN7{ys z801fvfO(ooK|)XhhMt6v7DU@Z%M^?P=5(RYf}I8?U`}TgLBL4(N|NMWCnC?NM~zhi z2FOV;$ciLhSmfu!aS`M7h8iVeQptZo8u4w&$Hp||i-!^7M&|IeL=4-Z!2anHF+M_d zOCrW92_go1vU-ptp&l18qBChg=&OF^BF0E?;g!0i=Rw5OpCDpVrq+^x^FAWR=n)m% zFP_dft}@|lGJr(32FFM%a)9=^``4SQ(Ker=2lxrZ&6h%<<-rq3fAL&fp&|>x{zHcG4F<$3mUT214bsM zqs*Yq*NbO@(W#@6vUwqApingnu?hAp4HIZwUTYoQ zU9RbO8pnLd?%u0FQmK(*x+f>oLtnu;B&&eJyh)i@yCgIFk|j5&GhnY*o)vBOzSyxx zuGL_9u-<81V$xwy#AHEj)J{6sl!R+lL!3QYYEE1Z*X-HY=G{BkrZVoOjJqT ziN8X<)e5x5-DV{n+l9iV(*+&kq4tZhr-dcKls}?0y8s3z9N&wd~;^DQ>+(uDei29I2zWF27@BmC*&!0 z0z6_{L*2WAhq+T05oJHtO_!nzhSyP;DNgGr=LwpSNZ;yCyj%zH8k z?0PcqOJ}9!HmL%>+m0Uym3DSQbc@mTP%asqN49vVZ7OnNY2yubGA-JV%R^39+EzZF zCA*p+B6Ag(6w9ir7z1R$KCy)_^b4R{mWtvl_U+7;)l);>h)==m_r$;9yX4&yV3D( zX!2-$-5hDU$w-<0Ep0H;IF9t#;}C-6&Kcsc(rl~>->iq1qu{~j&KYwc$%UPKup~lc zm2^KKr2~ZK03v=Bsr%iAGEltBw}SI+p>I`%q>`=*O{jP<@1toHgAb}Tx$0A)q!8Lf zlE&D(Ua}kCp4aLAB|&nhN9-9aShj_Eq^Po#z$|2Hepf@JFl*Qz?TsYJlcOZ14FZKg zZ-R-nY_nawqtF=Yu%KzNzdeHK?NOW#a|dIRURAuxyhI`g<*7;trrOPebM?jUsw0}Z zoLhCG+)ijCQ-7mi<|UA!WCW8J2^YzT?LkKHiu11pC#VrYab|X|dSu5LtC1kK?2&bn zM|CGvnJqqOMQgjPw}p;1x+P{YHE?D&Q4=OZ)FbM1(@YplWKk^0kX-H?$A`&~qe&DZ z@5^erzkB~Y4M?-arVQrgVi1JACn^#F@w@{^FL&98kRqe}2}qS)+BNRSU2RcmPJX)> z9K9<1PDAC6RIi}Yx;}?QSNx?8U6`2L61^+gi^9^zMk8bLO}N@)g10)o(ZY)%@sKe| zIiss8K|mdVY$c{Y<8c&9Wv&>#%4qjtBBQ0&AvESDF^9d_W_N(x_#JsVuy99*N~YrG z{Lj%g)CkiseEsQ3Tq)^vYV^|y;n=dsss3Vv6QOZ(*!QEuUJ;Gnv#3`L-&=q#FD|lg zY=48fkV1@~y?aglu7jhV4#?K4OGrx{8IW-USi`ym!~tzo@@5vZ3{j2RKTmJL2HKlJ5$&Oov{ z%|ffF=pgto*@EmLb39E(@q@VsTN-gdc9uvSKA#Ou4~7&sm=67-i!m_$ElogR>lGOj zP^sO!T*0Z6mftO9I1^#*vMV{d#QJlQPbGh-;CWOKYeT3YrqP0&h)1qynGq`z0n(QQ z$jNl6ga9)LfT#}CkFZNHJmy<`dZvC#h6?(csZo)+WS8SjvBjT#_i8cL8manPO#F(G zF8Ny|Ia3pfI>^7vbPD7Qql_{cB-U-{3jA472o(T0%AmJrb*_V21_2fmJ{6nI5;2qi zD-wX%mOo(9h*%~zHW2^W1`#kP`>c{bW zVm7)oY?s>fl|>h%NNuoA}{l?dilr&1(%6ZB3-ZMYq~I)r*% z9eAp%lX8WI+hBg(Nn*?Z>)j4GHQ?2-P-{jh4V?aT`WDNwu?n9j9E5I?8Y$&>CCk1I2ETKcUB4!Sc>SRempELrWhen2%^~E(#17~xT)>Oh@ zE(ClBx*(sQbF4oPWK?k@^^ zVwKDS8LkJ!1?Yla>u0+a1nbkElWjwPvBQzCaCz=Cn5;ZUI3b194*jap&+6M3x&iq;I`g$V|mIXB=&vmNt_aF|;#bH$Z$ z%Ugh%$VWS8w@Jn4c6plSItXqn1szZCRzVro+lb)^q;59;PJqVa?H)L($+A5*Ye+Nv zh!#x;=(*5h1}U>45*Q{GLGMFjUyB$?~3mMi8#H065Y8Lc4PIzVK z7}*deagHM8;1+C94(P9l6S}&}GG;AEG|GVs^lB{FJGNm@yX>OFX#BNK?UfVXHfD7% zOd7K~aKc=x!ijdU*!&8$SD_`M=3KbJW=6P7;9G&^1ws*%Gy4dZFMs6|%+~SQK48Dy z>8B#f&XeiROszhPN&*^6#)2ZYj~|(ixmw^MzHa&ZE6-7wBFfDHv}4KOJK|UZCyJqd zs`=jZ-Oh_Vn_}cB=*EPFWCmOUTC_9Yo1AJy9zd)O zC4u5xCzaiUF8zm* zCGe2b7Y60hzb?!pr2is4SwJIfCNNQ;;&Br#nip>}X%)TIGT`SI2rjs_iW628F`Y0H zY@0iUb$&%q6dfsw#WRR>%3!4^War7581w}{%s~Zho9CJ8(S1-3(1DZ#_>pozcO8#% z2w8M1<<@thnIqkJvlz2g5El#;LiPK$J z7?eUa6NDaw=a|CFW&eZG(y$=URW7a0@??IqJY)W&%HxqjBgN5Es7MFHs7 z`+0C&U5=1#NJ^d@sNNk)*S)+tt=?X<{_S=3+w0fAy*YKIOB*+)v&1;XI%FnRm>9tq zu{$(w)4F=HsDY7gxNhY}wbI}gFK~O^&}~nop z!QMW)oqj?dfWSh7*=EKI3Bk0@9hGu(QJ(Z%|9UCG&d4kv9-B)10-y8YJm=G1^CPyH z)*F5Q6|$-I`Y_+|iOg2MYec};;o_9H{iBrB;dSzdBHwhXCmT@9hl!Yc)1`C%4lyH4 zJW5|C`>8RUJSawHWY24i_q=MnyH9Am3KxI!#~U}=c`)@R<1HyQN6xI4Uyy@4jC406 zEs4W!j23Z&b9(wM8F(6bc~JAtR3PXGe$MTtNAAUF3l`1zM>G=-6H4X^m6?Dn1Fwc$ zgsH2+#0))IB^`HqmC5waC8kO|JhW-g)9GL6ku_9I|Fa~Y1PJ_x?E9Kt$s3earg4#v znb{`e^D7CIREI>u(wvrx3gUmZKpkB{DkbGzHP&|~)Av$5DYNNYr8>$F=x+L> zx*mTxPlpEq3clOC)DCmYwFFJ$y<`Ag%~NDYE7o9hzeSYnHpV4Mg1ZP~w6;~HB=L86i^rP!!-=L1$m zXVppAy1TzX{*d*{mLAzsE!D(q5W6cG|3nILfC&0eXOr^ZuP*;zNs^}gfAN2bKTI}w z&7%!Is4IA!*L+O6vNDw)Jsp26Pe*~0f=@hG-?~hmWH=O=ZNqO-k47`^Hb_W5hc*e} zcrg;vUQ3|gr^Y}+eSvi-s_0fEWWePUhp1d2gVIPyiwRCb-HL=_VJD$}i-hp;1QO~X zK9T+Y#(C5Au}3c-?Ep{DzkMw))F$2K_XoZ$ul^I$dmbajf9Uz zSK}644Vf)eC6;1|NLNGAmA(&jwbY`dN$YN4ckFB(iM_LW07%cGw?I+9+r^JY$D<$+ zVKI`7VWqRI#16E@49y9V4REJk&O1ULizniL%?W75)3?~tM<*ap4+HXg&i{tI>@rjt z=Y#T<>du5vJ_+nV{XIvXB8-GQ6^8#?h1Vlbr9ht8sQ`Ih7SOD!f4}On2EDS79?*V* z3zt}l3i_W4Un zGuJMVQNS{sQs5|gN33g-s)|t1dZs6wGV(Hl%IQdyNdPy}PG7WRGHI{|bs6m#S4h|z zLi(beSRzQRK?&`cI97m(y$!VEQV9AO7%N3;vAdq#>EyZ1foH-e6aaVw176aN|4ZXV z_{HZhsGt2(fPw#^1;gx8J0eT(4ML)a0j{-HApu%+B8{Z2elmWt{fZ)xJHtd~l#*SO zk{|j1i*hU9Lu%MJ%OCrcR&R}NKAwVm?~pa7V1+$|d$8+MU6@7IFXq=-{b63c%UKx1c*LWkBP#u7{*jrWyXqgV^;VT#jPDHu-upq2!uk`{K z`H#A$zTBw3NMd=DTJz!xms6Z6iW4=8vUAz==VU@vCGAQj8&M5mnOiIO|4B8=KDQB2 zEN7?RO7bUi8weXbUP+F|yKc$Agd4`796Q|MM-gtYqcc>F;YOlAGy$S`-)P1HC_|(L zu{jYppu7f+{Af=yZcKIu+)(>^+_XBdfE(l%#_c)TKL~DsDoX|;Mo2MCVU#UMA-e)O zr^jhSsaAfVpmVjIoBwh2!ET;><3mE5fVb}mPuayTQWs%p7eRx#@4O1Mvsn=8I5XR= z^?X|sG7^f-(n<%MLTE$?Y655QW5(_91NWDBCq&ykteL9?KfJGx4?ij*mHRFYg$=g~ zu!_+EIM}jy&GUY|Cf=edQ{Hep<1Y753t+p(Ztm0H>4!CsOUBz6pJG_}gX@3#dvXen ziGAuEYzwAxSLK4zlr0c3_HNB5DJ;RP7FXK`b^fG+A~NyF2XqI=gl}r1$F-v+icj45 zsc&`b_jOb;%z(?U#ARC)wtj^Bb06qI#af~RIl73I+IN^oiV4+X6VPT3SlaSKQc^(9 zJRFdWPRlCHYY3qXmKOirzJ#+Ko_cLZ0g?R@hv4AJOOKslkP3 zkM6NY)9~ns9B+&N0MF&C%j2xmPx^V<@AbQOT5ww4Z{Ke504S59sT~?u{zdDB;=ocr zVK^`+$*kbjtp+0{4ubXJ#GKi$iQkGM5HzQymXT5RX5> z^7fZ`DT+CCsBsF^k!3;v8&sgn?I!MGm!LdX|R{ zUdBWaI(eh{AwrX~8W=1;xWCiO7`}sP@gUx~)@FPI4wJ^!#@9<4sK z+Rznxry~7OgO^OpQ{Gs<-WmbgjFIMKXW}wO=pib{uXga_2m`k`OU_PyJz2VWp<|nj2{~M(EU22_- z64hdQ1|Oi~9z?*)xi!|j7sABCDsL1%%jdwBMEPE4H}~pq)N8zmYd7O7(^loRN0f!C z^5FD-RZml_uO+^7#@s!+XV7(1S||=p9c?uQK;=gjn4t@3Fxm-RK7J?gD4kp<9y;+( z9Bcui5wbA-VgLe*)D0jIc(4G32D}gbY7^dIrU?Xb2vv#p8kqRsTkUt;s&On6rt(LI z=iKp3Fhdjg3Z4nTzqZw`c*N^z;ko7cCM5AoHIkMH0!Z)z@YxlnyEk_O^viurwQDV+g1) zabN)p#^Y+j9##G7)cry%!%MIaN`swFf2wYb|JN2i0%Hgt(?2s@usKjXhi|@#iv}aC zwYt|J_aEM55rXyKKX0I7o|lKN`-}U{MNfp=0vsC1xVP1i$sSpDTIlta+A1*TU$w?G zblSje?eO>EVEEiJ@p7Qx>9_3m&8>d)LO(dI%J@up zp8lN*X82(dVuVGR8=uS<){R=Ov+XT%+o4};i!?9Vq3<@K=+!;O#ur~0J59fB9iqRh ztv=QARa)%tRmkM2>{4A|lFH>pC7Y?VYg_%P1!;!K{u;12dSsR4B zrtiQY*IG>-*B}?nhZ@p&Z%+==_dbc8zzIMn{(GSG6F~RT6YoUZd}!EF)3<0b7Q~wk zB{S|#k849cw9x*E+fX$o2n#zoARM~m;|-hfxX58MAoNcL2-hyOe;h#Ato_dPj{^~$ z01}QodF^*krv2May!}nt(K+F-Pn;9h7CLbpf^Rb<^iBo{_b#-597M1w5S$ZE0tiPJ zI`IU6&_5X#Vgb_OfrtNT2xec(7(1r<^Sz;(u1AmPQq(B$};gHOnGnPl>5)h=aB8q_R9Uc z_P}SEo7>?eM3mj8yLq}VJuu7gR^)fseQt}!rM)ER=mYxx>CA*%AI!;69!iAo6qVF& zqS&OS_<=kmUI|O;SeO|d3AK8Er$25BJ#3NH{GE({`rwg|LkOB!5GP6s9CJkF*Z(IT z&U^m&5w?!nrIsLa9$US#3iCbrgDJm$s68!mes{(_k)>DjtLh)}>d8TZWoc8Gn{`Qj zM?#5ftt~#iS5@)*P@3(HB@aT#SP03%P)fGpny9=PAMW|^MLlEizny7Pl^@plpB_H{sCZ!Uh(&8GfKACoQCyQJiK!;Hq+lt zs(1M47v6=4X8eeXXl{SEtNL!G@lLHh&kScS&JFw2v`K=-2#5Z+M$?nSq zwmJ%rVShf4Jvy4tnVe{k8I;On=49nT9L9WN+vSVSO&l>(Tz%8@I&rN>U_xqI58I|- z8f`7Xo&EA1Y~)}MgQhw}(#z+}l7J0%0kO208W!~}t_JATgOtmo`DzGMcC;yY&{$w4 zYefwD+dJ#GcQxY){q4Q=+xynPy}y2Ybp6{0>$hK7|MnZat$`Uk_{jQ#hZty8P=gF( zxk~F?Z8o?LEQGhDR%{TudEg_VXVq6!u&{N&M|}-VR`sY*wf$u51&-7W-S9*jy1s6R z%t;$GbW7dP%}=DE8|#K{-msxN>V|HAA`RVEH+1`k4c%Ecw03e0N#K06Zf0$RW>7Bs zGUBZrTYD>PzIYwUyrO#ds&(%!s@`3^?j3RaHC0wLc?~RCQI|LVx{4%U`vfTRHC0z$ zOAG7hk$nOi^oA`1i@iCrVZj^M7kpFI?aPm&+YD8#B%KmiZE^dtmDqSga2mFRwfK@P zVa+W)4$Ie9P`I4->P=x<7cEw&-+T+A)k|KLR{cMe9>)Nz`>hs5We2ME4hcf*z2iYSlQ1OED2YTmG37~WXF#rmn60RVTUtJA-$A@CKNR4XZG zXK0fEomwqtsEW`u=2pI+1RoTLVelHVXVXKCttiVIYx$RR_K5G~+&J!G7fq-%OT(AX z`MpDCntiT|u#|r^t&Qz9BD=C^Z4*b39Wm24)1q-!zQFdrl}BXC+5Wfkv~zSIefd#; zau&wuPhpu6WwMK>|K-*-l`D8kzT53}xouqXf-rqEFm|bW|Sic6$E};6~acIAoOW?nkp8$V0uN(XeEp%4!2SI}W zN?lIK35bCv53R8ZB<%bK$-xB*M*zLx?yp0eSO2=<{7Gx`HZqHB&?ey;wzrargl)Nn z%rghPr6UBi=VV7yp55pDxnI{f-@l^YM?PwOgn2yab$PW7=zitIDIbIbeUb~?G~;hv)`!a zf!mMhIplVIqwICM2DZYgar}|xJxX|~$Kd50w$O0p;8a$}Pklnvo^^ZxYjoY({DB6LJ^k!}mq0$~zEyXD%) zq!8ZcF?3$>UV`4=MCLu9-hcTHZVs%LOT+_ZK{gisWrAQbh2UE^m?Nn-ij>!~Be>E= z|77|Fxg!Xx@>*9MpL}sv6P?^|=TopFk;(>xkTGUwT}a@;8AI5cay;+>M^DnACDu?H zTs6*$92+^rt0d6hs@f9!qsDpHsovSzUCe^OZU%w@O_3_P)<+Xmv4W7Iy#Q?QA|fJu zMwL+BWF97=QUop~Y@1*G5wjj*c+nT38Z}{J)_hD26f&qL3OfrZM8QZ;MahGPD4JMY z=+0#kmSBQ=t zQt6$RSDeEHB=m+Jaf6dr=ff0IGQfme(D(7*rE~e9ii+?q(bJ^5e{Y%?$ zYG`35V9VnCzR`AUcEMFzV$}u4A;=|$9mAe1!TTP6i=9V;#@KnYj3&Y|MlU}s;rm7< z(ON=`iy01LW=QFiSFP{A;@Rp@>NnOfBCaP7bnR>RIJp3ovc4F10jBK_pR2&2_yK2`4Jv!==NBteV#65G!+zrbvxa?5 zGDa-0f}i7VfyFjpXoY~UKNLa>Kmf0p;+ftZP@I(8qWTi@<6;N&`sH4qiO&J$Otk+F z1VN^0-oO!FngUhR)_kTN9*JL?q8OpBf_T@dqFF_$q%gPOWAiDD;M5R0OXrmDhC>G= zyzGucRp46sKF~nkF7ah8kZiJ;k6al@XcB#Y>5PqNcPQboQc<2YCx{8Rg<_dH^Hd3U zw%;^g(ysb56M{S6@*`JWeeIF8LkBr#)KB;3*r9wcEp1Vz03Gr++XAP+92tOV=AAVj4~#?kD8=d9Z)Qe9GSg1xN7hOTJGXDA*~P zybNgWAcpPGSFZadl57t-aI9Rr>RNXd*92pPse zTqINkk}S0fLfwKyEqBbf{-S+$q}V$BMa`<_0@|2Bc@8BESbyky{tvUys{7wKqB0Cv z8+*lRX78{Sy~tPqNW?JOp}7{-R52XCOa>C1o6I0=sVp*XiL9rb$IAn|E_61pr1Y27 z5gj$h_17EMU(@x8RrRID=YQ3>{;E}V2JO)&%@6FVLxZlew)h+Ft%mUmy=<&P!i&C; ziS_OanPb04AwSOkgA_x1+SAT2f9|ShyUFx}S3=7%ByHW3ulO}1i9>E?(3Oj0E4;H`!NJCVRe_y2Kn1)ida?Gp$?ChJ8(-F6Rt-15RHJIk zdZeD^d`j(Vo@?6383C7v$6xJa9Y_TFhcM5zy&LB`GGxu7p5ZXrHSFKPP=YWKG{ggf z6aKq!2Mclts$?I#Fe&Jp;Gd7kwXH^IkjsjmB|v7Q`M*e{Q}WAieQVLb3g{Ct$tD)f z1&J7rh5qMuAu~t8nE=nH<++#M3TpZgET7Q@ZHVC?c+{Wmg-~*Uy&^X(DTB~?2n<@sXUSf1 z%Wn)Y3ml@KKH!h47dee%at#BhHcrsAm=K zGU1Lh)nG>+GisraU8)uOGNI3R{Kj&CWOY>-Y?KSV*-dY-kk1?caZWO>mPo2>UC9Gj z%K{^E{hSN91>1rPibV1=J4=x8K<7jc13iIiGkjKDhB z7r6~Z{>TcWIY?q8i7?`uuE-ZD0MqNGixj>Z>8lan^$(k{l1K? zYp&e^8M~$tDOL^Fw9%!LhmmvRKV667Y?r2B#_vnH$WL}RMG`DN8`^GhLGZGlOc5k#~9 z4@6t@T+@cM-(ggXZ;yQFH<{_Pk=1k7{qy!nc0?jEnI-)iUX+4+S-SCeUY=%`v zR%|bAa~3-?Ptl1GhlD>%)xPftPuaTe{-JG5Q;H0IRdVieTC!;TbxkJZu)Nt&zK5r* z@Lg{=NAh5pu2HC{_abM*Q zNZgCCRy#j_#vDis&X2F9bry|!;$K$2A{%|vTVxM5^%TjJ+0aWQV;C=y8BGAf4Oa#p zTts0)Z_ezNzgAenFN+^g;s%8XW=i(B>@}Dndn(Xo4L?KMucX7lk{Da}#JxQZNBYxf zaC}`a8`lXoHO=TaLbJW~KZ)KD%_XOa1S$d#B2~L~lA?}RoN@TO z{0})b6=STCQJ8cxc1~s(;^^XhpfNk$aaCw;jmLoV@M$N@=_pe~KC@U>?xE)EbF*}{>zNhM z3oyQ*?-$$TR4PFya&FQ7mKV>anjmwwtX?Dig}Tm}`Chp})G6}(vbaKr<6;D*!{>TG zV{VRx%zL8*EwrRMjR{}MP4q=adc+fW&GInI$^6&aUN#>a&0rn_E9$^NY?S6UyjC7y zKaawV+>SgLI?#yjaw?2mcH8)2>1@Ie%e&x;aacraC-@_@w-`@CMURvV-iVIP^5{U= zI>@jj3)m{8t%l&dQ|{7B-04;n3$3F>9C|$cNjaxHDZDoJ2S#a}&20cQsYXB5H*8)Z z<7%8#;n=;Qx1<=28c4(PI##3GU9vrVG6n-&1j7bsg8Msq#*>m=@eoRMUs{2<;){=!6^{BDksyQ_c^igMwk@K(Fm(dDqo9C8D_Q^QW`c> z6G@mxboFnhSla0oh%@GB@x&SCVGwf(Q1@SYYmwi21#=!^A|%xl%U1wq2RF@gr#MOx zYf`;&X0|mZ8D2&DI`EiJKWeg7S$O!^9J-|B$pW^7PhW*ns-3>g*VrOvu5#unTv2_G z5a5u@50LwCluDT+W~vYo1oVl(aOFvX;mNi?EsH0meW`Pj=L)XOtsHZ?FqeWk%(<&? z0-)WS*aK%0f~mP3!UoDj42g0hkk+cj9Lm6Wk7(O$G8Nf1_T;*E^7w+IEed!L>Q>1!n^ zf~{Y1lW3H-qx*jR&PH5TZCGVOjd#xYgpNDh=z!T&f_0&uN{E`EB3#`VU+H{gY4O7x zU9B@LrMR_6XQ`CgYWkV*X*~GGbic!K{HYO1B?EZtPFaV-ET4)Cd2#eiX~(Uhu71M# zOZ9UeAdW#RWTvw@ifpg`3EG!>6n4++k?zLMN$qV156E0>jekgn4L_#H(U{C@O;?hs zTUAm>Iq6D{`AaEc@AQ7N|7fL)Q(lRDktVppIe+z`-}*RaW;|&Xx07Pk(D4!@sV&}$ z;z5E^-U+5%?_|-FMImnnrbgi>dz-~UFNou_bF(NyC^T<5O_jrOi|&pXTtGiCEST`p zB_?^Qirk1gMt6h4n=SVyo`Ne*?Hk-ov=uaB$vYJLs=Et43>J1tIjJJ392K*hVk;mMM4B5 z1_=v9ud|NXwI#Y&kYmV3B_mCms}9A`2>f3$#KYS@Aj(#Y00F%vj6WQADC1(6GlbZK zy9Rsk1wJ2V4>Lk|z*(|jSP6urG{hdfVJf4a7JF#TRtXfnSp`0*I-$@bMG6XJ9sr|i zeM~AM!mWnUT{jHLAxR+s$0$bpXIUfdnBc}pSZ8aDM47+rBOeSex6=~~|K&WhRELVC zvJN#KUTQr2G{++YvT9nR2~|QG(P$L4(7AZdRTo=K#9H%Q1O>*CgQAg=4uzdU$50NE zODN)6LMwqL#raqUT*3ML)#OGHf&?q3_e`cHIgum63+$_@=lI>-w7Y)WQ7uLNG)8*E zFswTkU6e|s+S)L0IzYzOU$dE@CEu{gO*%>_k(NL)*)Ew0g@{8qf?j2jd-ZC=|5ILT zlk-I7X+_f#)n25}O@Gk*v&5>&^r9ImNt7f%)ys>Khn!fg5RRg=Bd$qnm3h0t}6(0VKMOa+POA=F9luQUDWP0I={vpVcLktb7)_vvFU%d@8)8 z8iI|TqAZ#r9*zQBPy(`2+FxyZXW4~lL`{AZs350pBIWeI{CP^FpB4^Z2{j9v!!lScmFdD%^053W347_LBf(_fNGsR&9Usv#O`d6j z*kzT>@mzQGAI8~8p@sSs!w;EGl!D5A#me=I>aEg=Z3C7%qCK^$zLLC@1Hr)&6TV&i z5Y5mF5gD0glgyp*3=^{zf--UmQ80=_YWf*DiS#neodAd|J#~WlkCj8Cs8(iwi?y=Q zsbFyr^&pv>+rY~77Otte7Ae0xsLx(&ti3wInt#_J&(XyMOX%tux>$AqN&pC)fvaJ} z)o=kzf-{%(@#4P&rV*f(1f#-myq{eN&g&|jrL8#0nZmZJk5%||0-cXkWys;vcE!FyKE3Z5Rm+5MtMFLlEOHC`J)AVoWg+2~9oIG6mO6`h67vt-IbX(R~|P*P^NfJLx^Nv33x`WOetx{q1$ zVBcx{IMS0FC%y9TXkK7~tyBTzJD9Em9==70#H$dzKy#{P_U)2o!xtvo!5xmR$jS;n z(T0x(9)kb@jW71e5fxrwzJP#Q5dOm$!pt2&U{NW-=tq3Y-HKCHpTKIMwqQ!>a7W#Hq z^V>jyKu#zEsE%C|792#SZXkj-Ku%K%fgFYUxTH`FQe)|8{N;vl$M3x$-0AyMOS~IW z00$jArg%s1xJcMgy!%@d?=8`8a22-$B8a{g!ng%NO_SmwQvjODVVL^yf^>;8+sfcE zH9>K67LanF{ej!V}7nZyV-9@cHht~CmYM2W(Sh#A07NAkn zQ!&+H|oK%sXs+?Z(E)Im4)4DmCNpJ>)fB} z8NG&+!wHbsO?EgRlqoL|w^ilJknr2M5z1M4%_sSt-u`yJPR75{JC+gHv?;u$f669d zOJo&(fm+{Jv+;H2cpq!vtS29nXp{D z#tBf2q_J>*nIHH=mYB?t=XfPb^9lBn0NDkbnZ>7%hRNn4zg~4l20bX$z;@F zu`WH6*yun0@@L-v+h6^!w?Dvhd)oW~e9Y58`05tC;RnPFrT z`HISS*^6oTVo>J1AOJQk-)k>U4=>n(#S10GC_iQ|Vpi=Q>9#X%X&st(;BI?43}12V z6fZ4%_wU(@D+5%a;{8AMnfpH|WTk*!$;Kh)bR|j3Z+!jZ@9&{< z0KDfQ*Tb*==7;@Zhlli)jOLBLTK@4rZ}(M%M1TM9)#Xvr%hE0f=yY-e{52-_djW`Y zm@1#TnlImBuOhM%>$83Hv#l2#8N@e2mIm{{-5>e%H{bs=cRb8sa_hAL=0^J@=749> zkxvXIAF>w_@$72ng`wo*_9B9kPJc{#(7ZbUkWJF@cXUdo>k3v`jb%pU4EDI&0EuB5~~G>Kxu>~ggsamZDjrw+R_r^pMPl$qLESS zRAZc?SAl|h8A-x_5Va}MxZcQip$}(uKMti;Oih7jM`69Ng@Sw`J}nq@ z#539x^?Bk|lNY6OW^%lv*}GhlxIJF3)t7M#fTAzr>y>z=2L+LUFWi$|LiHVNQ=O(m z3-sAHpJ{)VBCTEvM|9WJRp^G@}Vu+aBNY|2(m%1GlYF|2=DErF>Ms({HL*C=Bd-c?PhfWs76Yth#9L>ZddBP48Dh z$eUGayw93z5<#xTJl2QhCe9TGRPci@DR0qbB7=?3rnuetVg1sSKJ`wu_G2sP6j?ta zHN9T-5jms^rCZk60aeeAtBJMouxD}$UT zN%{A@92-(21>n^Z1#sqyQ$TstfP9z!PQMn$MBN7KFzV@VH{gxuebc(Wu|Zv0XfmYz zrYg}u0Y>&xqe>7qafE?B*~vl}KQr!(I2Yt*qJeGR{MGk}oZ3UN)#}Y~$1Ef-M-dc4 zauCo7^N{iyql@R4H|w`P9KIZH_yO@?ci0F2qP(y9fzOe%y`K2Nr^rFLu!^%6qqi0A z>U?LOA0y}neELv+{kFXnW;AKJ9-Wa$(D5yTGs;@%JoPUOHh#TGt{Z$U9L7mDCiE9N zO)*Ra-X>yLi>}ChY!MFv%KQ>y)rP$$b=>EC^I>#;x@11iJfuF;;SE%3RQcM z4BzUoQ{-nSo{!pToR8Y$$Wvw@2s#c%rEE@F9#LP%4@OnoksXZMEY(pdUYv3{6OM-x zeW7^+%fLD{>HAlg^DH57&1$hzYrE2ZANbvmd!9_u)21-q=Jy)BXH3o8e!#Bz37m-EG7y_6at z8m_4o{E%G%T3SIw=w>)dO=ftnH2Kn#;N|4pwtUgF8S=upIIYDvE0Pl9i`AGIhmIx2 zH?PfK!D3daD;T0>cFo8pqWC|F$k5{5QXFbX>R(-O^-nR2bnm(Pw{Gl|idMi<&(%M% zZ&LrE8w=_`49d1Yut+D#25S^Kk`4INPLym5Bb4)zIu%m7i>ghP9&4a&FvqXaeUJCG z*7lu8+4#4tM$*+&spK{bR|HM4ix&cGX1y; zAbi^O!jV-KcRgMeYnxS(xJ?W>Y3)mW-FV#i;7)AZ>fKu0r*BC&Py&+Ic%%cY&7|oz zbtZDnbSA&VOcmOEW700$PrOE{YlI=ThSAp!3G=G|7D4`SH&dG8?~j#TZgbl~0iDsAL|@+do$t$_)fdYtwLxFXcYWu3f|IQQmCbr$ z=yo|q5kVIG<3w+M%PwM!0}CY4{#@S%&DCz(7cI>emjxa4xbh${a2sPPPv}#86pc@r z^MKgnY!z;Ad}_)zbdDcedETI|eQfDaBc3bXMkC!?^VZJ|IvE?Xa8*XaZ#a-7)fWF1 z8T0*A8eKC7z`!DzRu80byKWVw*s%5>#ADV~K(U%~$#zOHo`N|gLkEj5%}{ma49*ft z_)kF}wqfc3KSvW3Br}@))_o|9@=?4$pEM)JS|$r2dsTku(sHEr`BxFH#?K30zJqlz zrW!Bb!4d}PPYx`ZhI!=@?yocR*OKZ{|G2z_?e*EM^U z+?LAB&}>tSs=@+O2v)LaMLpF*AwGvlhzsbLFA|26g`)_L5+`il2Qn!(QFOr~iWsGZ z_S=9`I%45m;)vdOu_KEJ0L#!PdWtdNnB2S*MynXJ4d7RRKo+S3VnSNi@{!r&H9#aL z>Rx44uYyca<_8i$Y)8AQWC0+>T$Ph_{70-f9pr5$9#$%Rl{BlA+eB%Va-WRRy5>b@ z6#L9Bk>PA}vRrer4j<$`Cu8yO@`5>8SAF9vSq?Te;}cEed}@}*sTsT6F89K0YG@U< zc^v(%XK1QY=ith>nA4(5q$FhxumC{>x+=D{!)R?ln3zSb%p}2B00KMa^n#py7A$fW zL)goy_S7o&qn8TKMl2D>h<_7vgEtqLklF{)JcW+f$f>XPt?_DLy;R_)sv|3NQU z2$g9}3{z*9S;@6#k5}u>1Es}&^2~#+=ss+KhJAn%A%pF{@A*+OY9vvOz>Vuf|3)@} zG69*M8B)Q&uFV4!bAMm*{b;^E7C|xmzBkkcK~p_vh#1dlWbWQu*>qK@J(({HCP5Gn zRCoo7rKRglZAzB3b^Ua4yqgSmQN1wOb^4vjnFdP?e;8~~^<5k6_i3=`bsOw@wT{AN zu&7?>xG#$jBy2L+px>K=#c-&U7TSBTf%X=isO|u@b?9qr?q=mpNWNsLx1mZ_m_xJn zMR>Kk|2 z`bSTxYB4f;&P%md5tcRQXk<7c1+~VCmUI|*-RGjdZhOd9pm-=>5&5%-ewViH+q8KQ zaxWiznO?d-6lC_Lr$V~rB6%F6NUcvR)$EKiLrMl)j9W^xb zP~g{pq-dZrO#%8zKe$gV{q4Z#1xbqh!7MDF^V0cn)z`xKq{IG`Vz_@kj5CBK>RPs* zvEb7iLpICNW%<~aA7G}er5IFuJu&EE_o9Dgc5#5}D=D(Puk+SoY2SHQUqxyN?FxWM zm-MW6>s6LBLXq*o%A+}Jl7^05^A@(o9AC|FLeJRBarVW3LO9z5Uo9*>D`H9W5!{T9 zkz5%0mu-yjYy*rGphna8CR)6zzN6qcs&W(rrnwUYC`bUN(bl`;U>Y*9iD`y4(-@zO z0@D}=RZQcoWlY0H-I{4)DQBADv6-fqutshC&|n(o(u+)UECevUb@p&*k%~^NkP7hM z(*``x+JJZI3g9*IT@QGjyl{L~0r)Ojz+85cWxlk-R#YwRn3(eYzd+BX5kWl32^hqa zO&JhRN+(f?C!aq!K7x4SSY_r9tx`U5=rtg1y2r#5)k`?#33d=;^k->h*k$pV3*e%& zoOPL9#HY4^jqXwvWZ9=@41oC)zbG&{KM-q7db&bkp|ea{@(li9?kw2)5(uzN_KhmL zCf6JvsMn9E219Z?Np#J|BY6$C(*yG@#nP)U!&ypA696sv(huRgQx3`05&J987j8st z19neCe4%=ZIZ;E*^_Pn2tMNyYsfL^7V#})wp8VS#RFB1Kv;A0Kpooo{o5;G&`Et4L zeQcVxaMZ{&AwJ0(oZw?XCRg))w)N99t6oZV$+H$NGk4Vp_od3*n-pV=SAHgP+BHnr z4^L3Dd312K7blNX&Hj(1WP!db*bQX`bKyletU%jMx+kg-BZ(VmH&TWbV1oXuL2CZC2-3U-Qqz5# zeHldMQ%l_iRcF=%W76az`Y+O2y%8brS}drwJk5_stu=s*N;9>I_@6YDmqh+R`QHgj zsI?PL^{5wICAnr;GR-h4mi{fUWV12^;Iq*nsZ3gdkXB8sgt~H@>>FL73J_>frw7rGJ5v?C7 zRX~6&cayoTg0m=O1?Z__3yI|a-X&yo*yb}K7fK`HUx$qb%|v~wl_H^Vv!GTBapzNI zd4`g8*!5m!9QuS5KBzPSM@{0^C5W%;uU0#Hm$UD=U`glySuJG@vpZVm%W}WpPl&bX zvoVBx8&s+|W_Em|-AiJ6!UIB0cmRrQfU>Dk1gvNSHh{nsR_MmsCJZaad);Dg(LZpv z=&`?Kr&AU6_0i^MZOGWsEMJm06Uz2x3&IAHLj7U)yZWQ3Lt;L`nR=_S_~NRSuh&;W zdW=UYI^+9OHuP1{qUbi^566unHuYt|vK~z4P=5lfjR^7paHyn-;|AJf{z0M&QlnR4 z=Li^1*JC&oJwv$yXe~si=8?dfmXpY zr+ezgqwfDi-A46xr~srb(w{3dB#xZxiA9spA}_TzxFK~nM&46YDaN|O^YE&uhFLtJ zXnipobx&a@NeB*8*Yp*~OF8|{d!pchQ!uH&oPwdm) z@Uu;86XIsR3QG>utGF+Qf{&p7iglL_z5@2olUuUmQNd&#GyFeq2US1^=gAjF4s{(! zK=`fVXW>N!&}2&Sqs_xa;Azh(`xFsaig^(}8EnBFKP+z+tW%uz!h~$s@*0X}NbC~^ z!@>kWVRXjV;`{*p02#rBNV5wewWM5T)sz#zz%Sv)POH!oj7?0A)mq+TJob5p|Ee`l&8L8m{H@5C@MC%9%XSMqoI*_V8!e|?fufOB4H z*8$W%C#O`wrc0AqqYC41CcHUI63vlW&~N|)ZUGo@3$kJ!ek27O00y&;6bJQLXh#Qd zmj%+G=|wFYbQC@i2Ao|$fFd}(t8KPWEo!^3jiQ|3z6sH@;4GzRL=S0-=m$|jGVC;hk{6GjQxAr5>RQp_JE|NY(O$PHV2Q2HNYb(t%A68jQpnJ z?ys`?7${aYg0yqQ8L$wh6^6McCch!))T6m=?@A?gA=iLV0vDPRenE@#g*HR_fHqLx zdJlm|Ps&vZ`6RlB$cBD-ma^fIfbR$A>@~R@ z8>&tY+~w1;CkGyt7b!V#mu)PF{E!tG!J&@_Asz6`b#>voq`zQ~sj;%hB7X5s#gxC7 z;A&#YcfGnc9yX*%9&d5jXpoAJ%^SPK!l7o@|H9fK*18lFdS{CVGaZ5`86ZfT5;!YL z`b4Ti`Bex!+{O>OMQi+q58@>F#{EhT@ddWHkX>X**N%f+VXVb+C&@CReeBfU9Jm4K z>D;vhurADFjS^w2YI9Z%wA;y!(gfTjmCh>K zOsf+$MTm3C+Ygi>EU}uT6E(Ai6E%0c+@P7~sYuA)rM8PYwE$8;M8?Bibs-T9Dv3Sa zcA_SHY|ZhVfse3~(5C;ZRN9b$s^0J7G1Rj!7%6Lp6OsG7IPf=6lsqfvG(B zQoYbCj`_0TVmEjUEQd4#rI?fK6Vr4WS)O&So)cgM;6}d=L*w866>0HH-pDw%mgtKKHPnz0p3fShk-K!9jj6lH6Kfa8?H zZ%4(lV7+X8rBM2k`Kf?xiHF5j$`onLLE(>guNL|=JQl13sHy5msbeZ>4Nh=RJ9w@3 zQ+X2+we=)i=Vz)V@w-%?Mr;BY41?8xmpT>8y ztrQi=Pt*IY@%>iShfwwG=X!64b~15g)4T=wQ##gFEE^G9)2un1lU)M7a;=H6Tr>(@ z0SxJCnO{1$qYqOCm4zIS20(!o!hTVDG4$;wqi_v5``{-?jQPX@q;iEgX*Dflht4XS z@+c-8kgi{~u+>hrAb&K10YW)ErJ~jgI}uOC>1p#tES|TXGO4FG@?3+RJ|SNOciQ5! zv}aE#`=|WSCWq2l?m$HC^uK8yBvWI%PtONpyd1_u_Fx+y*Qdpd*_Cl>gDx!nVxWsu zykPc-69=aoN`mE%yOdLw))NP1))7ZuA=HV3HjYUgp@Et>&|5|vxkzbq;(#E`0wSj% zGK)KPWA2JdD&FB;ZuI4I4zQYqz$|mXa zpj*fu8g&buM0AOQ;ahF<|3UqTdBn2vNy*&hxtI!Pg9Hwwt69I@v3|xPczNw-)sNlf znttP6=<<-=#oXG2Oq`RwhQ=hV_`zpk8K2cAhh(S;gaA#93r;!1aw*1gU~n{r3ZgRA zdcEZaqqIYQ(+XDiHXHj@%^WmntQ$CF7DgUrv+|-w2)Jw2$BczTX?;hwXj67Q^$ur~ zf^Ri2t9|CC#WRqIe(0v)2HDv z@};ZRI$~);Ky~nlwTv&K=+F zj2-5!PcXDV5Vji%#gaa~;!DVSlC zoN6$015s7K2s9I6;5Y*Ye6+<;UreO=(3e7+NvT9Okts(tY}ngcwZ0{^oz~i_@nx}9 z*t4E&G3@1`CwahL=#hFNq{0ao(=ClsJ)}AEkzN#x#i1LEayT%nkg*6duTB*X6#vOu zgnm12E+P&gNJv<)4Ak4fTtw^Iqsm+~sIpF?@;QSx$cvP1ympx7QgMpYG5jp>7^WnH zF9tdkBUi1WOF+u0rYJzE6ZNT)s9WC~Fiy2v=3C_xku5hAZAt4ZNzKfbgte;iAkcBH z2)>eV!1qPKm!V;jT|{3I%5aE)go{STYR(acfdT z7M5wCdgJD#0i*p!-O*#&lg$q>D2HMIsgyx9DA5rTwx%f2w(7lXV0>y7LQhJ7 z2%Z^{*w7Fe8Ab*LlOSVx+y;{5Zee4ARG}C5Y~LYYwf+yxj)|PZmDH# zaJSU=oPEyOKh|Eq_F8MNT^jyy7M2LAJJ25s& z?IdDVYA1^(EJko*ey)Ho^Rp%LH8#~~pZ7HyRR>y4>Xkr0FV=V0J~2L{(;D&{l}ZM5 zf`oglk#O{tm8$eYk-F4Om^;6sTJbbl5D~$8V8jD?+zhY-pXM%up^~rQ!W2;C^)O{1{`yW}teYX={B=zcdDI(%7il$DtfpQabfx4J>9RS`Mfv8fbZmvI>BttV$fuXsZ+?byZr2Jb6JhyPhb~ zEG`K`=oAkyK~6u(B0fY4*m8lo8HnV7tCRK+_KxpW`m97V3cMMQFV zX$nDoU2*jl_HV{VeXN*ityQD?3KOqZf;3RqICw3nuX1YnC@`9$zA{0kzuqjhbr<~= z#Z?w=B(v05pe{9*xt8Jz0%%5T3F1nvsj;m&UarOxje8+#tdhh5VyxYt5zij|qhcmP zB>0Z(9mKZQ6+oe!{DjtT(=4WlGA=9wtQktnWAT1~fXl?1vAnP1j^Kx!{-O&U$(Df> z_g^QF>oML;0f04Zo4Cm>8VM zgy2Qmj19}NC9&}cG%Ya@0ONizjg$4Z)B@pEkvv$u;tMkf#t^JwsDGC4?oPDs&4OWY zAz&~FFnsbw|3(}>ewwAjr#uY*sm9VgJA9^HP?#P5w=gC~Z--|CP9C3s^jG`f8(YS| z%9~U8bdE*ttgzvG_2UITfuGeNWP<=9CjZxIZdP+9xtRQ^jV`Z$E9Fbbl`P@L`h!1w zZby=e?1c3Lzbu>&fkJXa7EzcnL;2(Z)dxZu<_K@5C8sh^{)rdY=L*ZO!1(PYo9IXJ zvNVS?R>kOZcvzZL5ciW(F_5-Mb@8uQbr6gA$cke&mY-6C3=#Fdv-rtkiDolf8)a8` zBD(%yB^%X`h{Eb4^{0%|f*lCWurt0~2~|7e%W%Z*S}NAL`S z%Ak;~;FjcZ38;J?@Jp)C=G4Qa#baItjz@g+}CW18WE?lO1>E zb1v~4n!zHq&kspOr>zFoNUU+J10!Z@Td~@D)TWI^p*Cc2s11QNk^dETPHiq+Vr?X@ z_A!ei0G3H4iP=GiXcvX7`H*RA$p1!HSFZI|M#CPnAa?)(CvoWRCeQ(Plq`dB>TH=h z3xvrBqRuw(tw(`?HD@?d!)&2BZA5aJ1sg+UYgGCTj=a+ ztF!yton7Cfv+J$S(zAAF*XZniGS?eg2Sefvx)r>Dlq>m>mc|cG=tvD`i~~e!$W!dz zqXn6|IJW_(q_Q+XmFEidAW+~KRvLbeOvi6Ri^gTB9}G6P^O)BjlM95?^l;W?RnC42 zZ6L5K5`t8^f;Qb@k6_A?1L5RXi6riiH1|hze=chZFa{dm3SUsu)vQ+eL1cLw9B!L& zRn~v>`Ext`5~%m}+@skX^&TOlj#03rW0I3Et2;Kd4x+f-HVq9}LfEsbmAVM4nR1Lk z3waCUSYsyaYqN^hHO6IG=ALgW3Ytlj7|2S;;z_koCfbP|>8a=y9bXM^wo{R9w2@QcL`>cRR+YQ7g)Qn0rDaSQ!dg$*j9Xd zweRp06XKh(<7GE`Objb42yRqi)`Z`}q~zQrCI%CSub?;(Qc5dxO{a_vqVB;QKO$+5 z#X_B8{cd(^Frz27jdwz^c!({{8D|t~&fuQX_R-=rXXt&JGo5mXe*BawyIl0ks#EHz zays!a?X|tQ1A3+H4G8%tYDGMJUX{~%QLRTeaAidNSL=}FotAz#6A{ls2Ym~j&eblU zHhWU_0G4=3p!j&+q`IaDfn=B()-E-F<4F&cpwt;pv6#{32{yLmsw#hLR&tE=t8z^*m7g#^+j zo6LMFr0nL~nhI@5;EC9$ulQ{5YoWQ4)rn3jp$1EHSSoVDlk5XGXWd^S9kK@NLm)||e{%1MW@cBvhE zn`CWs1nT_D_09sqW;AP|E*QelU`?!f41?SnxkE1^EmGgmx_v+{!&OAFY->5dz`+4} zlkX|hZle}#n1bCwlPYMA%LOtz_E#oDY>?(X)>2hq&ZszwX4*lQQ?pv0Svb2J>uoR& zV0B;PL<8b2Kp%xgjeKtz((KQ~(&LK;zt0SXh2$;C2X>Ys|+84?vZGd7dk7vQ8i3O+~UH*fBoM@3BP&-~E29W4zeCntX(C4`AD zlTL%B;!pC%+=M(VytNu*v!=(U2nELTg4X#vaVweWl>sYVNLW@Q!@{x}?F?h1}KwCbu7I^%@)X!GtJ9oqRx?XD4o-Eh@LsT*0BKHL`M!+rM((w9t)3IffT$3duDcy zgMvE~mS6$ZH8m}4S8)vMW$w~kDZ`oU)Q`COb96T{j|nYUrYHfvHE<_?w--V@mQ4K z?=fBN`zGt7`UA4PV3B;XZG$lm@Q4n^ z+0Mr18p|iYAVlC6ydccfzPc=)zcgM`;eWNlf19OJ4q?dapZ|AiU@H~c@n`?R3Y+g` z-Kn3m!e3~xZ!kU~JETRZ4(44cGM_0GM@Cz;p1cUn?uo{G{cuKzF2w;1z%{FHAlPX!+!l#`p{~FBkGx6>4zHp&Ai_K&ia%!0)(Nl9?r116D#y*RO3BN*Ga9xZG}6`t2n-u2UwI*6vg;dLEBUKidj;PM#Cg$%_V}>-ODZsAjdbDnTvq360N`?xnQ` zd`2#$R|6fCYr~vvx+R_FKkD~a02mbo?oW__@DlC~|IGL&@mVW0+2tJB7%t${3rfny zSCxHbT-R=RJG{jU6|RNu>6?IfQGc#QMwOA#QCF6cu$}th_nzC)TSsTeND4pN7N}Ur zARIV(XY~C!Y+A5-xmPwt<{QKnL6Z&o9PW1;9vguuF8Ka*+aw9;@0hkq|&y01W5X#S@G%~^~5n;_gp zduUzx-9jz6oLz~sQZ=+7Li7a=?~?ZtAv9@K`@Z)%^)%IrVxrx)%5(xld2gqczA21? zx1MQwi#BKCt+us~)jx%=g8!^>KlGrt5e1!=at;Ev^9lr8(_R39_`^+z*wz@TaBiQZ z+wTS-q?-JIj~0gzmvTX?_akpe@0;Tg{&iFaOOINBDZx7=;fwnB(}|5DbO|1?8LbchWcQWA%fchqh`f~K1gZaVesCafa zZ@7Wn7sm#5WJDMfTKEwv6^XGa-kdDbmP<*hb#mExhq&k|iGV~?W<4wPJwManCtt{Ez= zUv+Zw9TLY>%;>K25HPOTut>IqxQbI>C_L(q$7&7V52hMbN~2m|b$stC&YH>ZYF_X( z>ziu2Pa8?Nx^jC0VE36@YCS}5?~iV&jcT9vxRoL~Ii>b*Z%Api?R`Yg)86)GbV_Z= zo5R@G*OCs7^4X^Z#vux~e>TJi1G=3&D3aZ2$|a7qyh3ixpJ5;e)b z6U9zWWQ>~N-m=hhGfC}}W<#nsL!L%4GpcUFS+&D7bn0(ezk`22! z(HQjCPd8Ul)wVYk?Vrg*Hj8aT&LikM{LbMHgD>@QcIr&t)9vBr^ zsnAB6_lx?ctsuS)yX{$34~ydI%~kdlbN<`vrp?dQ?18Xz(-^{Qiw{6wS2FeH!4wgt zs>bfYR9WiZ7=}tO&Lt0Kp)47%za#@?O}MUE-yk?&N3L1CgO#Yn^Zh^deZ*a=N^ zwf+{}gfs{1fqK>@52^#A@dKis1Jt7JFQD};V7&bn(+mXrff!|z2GyW88@S0k^iOcSqsj4^-OS< zpZif^I}GNhMF5-E6p!e4O$#>dFAzU4Z5F+`# zhcZIywBHtkgtBYl{4M48#%- z7Y{8T$fdlFt2G6dQ1W$*?Ty*CJxmPNIDrEzz-e5+Ml|lg3UIneMl9x>*v#1;=jf)(S)13MP2oGABi}@@fLY{aR-x8+wlgtb|4V;7(Nck$SVIpf(UMNw%}7^a zb{q9tJlOoB3XRsyLZA$Qk=$v$xVijAUJD68Lh2Bqd(a&joa$*+;;$~WW}pnO<=muB zF^Z~8)y36J>ih*%2b`Mf#4l2HC>Ju7Y-^@*73MR!s7?=!AehNQ=}0m|PCLw6zX??PN9#`pW`ht7exAAYcI+ zgH+pnz(1{+5%{$nw(cj8-!1k*mB_DS1kDvYA>z!W6^1=4tE`;>#s;mgv><0x^xu~j z#+sdm5QL3SPU<76GBX-9C@W@|PrL0({aMJL$g@_pkk{n5L|Z@!9mYfL0xYE;n;S7L z(!hv3B&yo`GpE`s4W4sO*cl3LwrzY)c8k$vOr=m;{}|1}w0^~&Z2FVg!slk)aEGfZFxyw;!B!-r zKJL1OnQgNEu{~!oqUU-|Ipk&E_P3Lj83oCG^MF=O=iBIY;GH4n?#20Bjv@==h^|^Q}2VlcB?KOaFB0fBXEo9rZ7o zP1N6+?lp62F1*qAmQ62~a|r9rTGyOfERXW~8RG|e^2(T2C0=YDWb_@b$Ol+|q1DwZ z%zC1ID*w4vy69SY_`l2zzz&y|r6<+TIKBSo7HU%cf7~kdR@<(dkL)beZ)#Q3>vsPZ zzyHvjHEG+4&@kV7wRii!u`s?iHDa~f@1!IW%iYa7`d?DoR1O*v)lSIe%jNT&I=t^~ z*xa&*fDYz573d6ypj2S_3GoR?NHS@P4Lfq|8%;iX8az_uf&bP3DN5@Pl{%FA9*EE~ zNpkR)sJ;)!rAVsG)FC=uxgC>Pj~oCx5Jg-e9y?-Q3j$ehf0*MDw&AIK*yo*5?0(5* z=sufPt^0%6{jd>7`f}(#dmxCrQDnD4JJnJ&wNT+~Ey_T0m-hRyu@mGW{`xU-Ql%k@ zb+PFaxT()JDjUI+^eeWjkAa=635R`QB&4TxG4N_|JGc!DxfdYpliP`cv{Is$>>&?Y z5EAeLAc5335)s4!4okq_Y)MdOLp%Uo!xW=!oy6UTxEb$Bs@8fm8vRt6u!N5!y##gt zYu)60)wO;;$cl2%VGYf6nfBEu`=mYyw_u&XmRlojQx`9cDduMaB9zo@uY~PL`o?qh zw!eu%Ql_94OWvU|A+FWR^1^yW&Rzgc`j?C8mteH4<5n{YQAYMVL_iU`}V@q1B2N*0GM z{*wTMnPdssq|h1bOG7peG{MR7Gyu_KTq#6B)SmUo68HKmJ_JKn zDPN<}4-}=2xX$lw#xyN`iaN~%(H3EV_;>>6mb)jgKHS^}-0u7ou?`Qc&#cpEg@N^@ zT5y}kTBvZg7L*%JksbV}De8tPYSpDFQlt(q8DDGWgO8ixv{;)o`K4l`8+nzr6ei$T za&yBw6k6QKU`jg3wf%Xo{U^(c41+V>Fs7um=;`LAlk0DNN-LKZx&CT-5#;)0BG)IK zT>nIYLQJPKSM_fv_YjU{1TR9r2KS2}4!D!+!z0CQ=lkH4zki#*{x}WjvnOPhFR1~} zM0nFR#pCd($$P|l^tgFVTolmB{%rPJ9-hm0XVRqOw{1h>!*=D(e)_!Q`mB$u_;eW8 zv*-Qd`2dM$!xuZn`9N_7wNIVeDZE$rpZALg!WZ|4FWy`D)`|!DvH_rE)iuR~!YfNu z5)Iv9F}_St(E*FqSw7K6igrUb1$@v*y#;AnKWYyp3LY&a3Rs4H!H{*mku4v|BeH(j z84ckNiFtnIFdDHLOL5Tj9nNby7QDZ$V+CH+jk4lWFrt6cZK*}hj`VM!;e~l{&AC3a z)9B6;>0k;)Qtm{6uubqsA@2$T5ryk_N5A$SQ?h{$UT=D$-g_!mGe_bN(Ya6cd&x*)3 z7JL08>6zfG`T8{5-AzcQU6BW^1VD+l=(GTR-Y3gucO6Y*7$qcOO!7bjAn~_-mMeFu!iknnY z6%SF-+qtCV#Igz4Nkgf>!$2@?K=AJ7*%S7xr2%|gV}o8?21n>@LmfC-@!m`#8U-T1@Xr zvcltP^UF;+-~6(YpPrRF*4urWg_8ew`&*5qIuH2=SysRB=jV2^?TKF2oFRbM_4`6T z-CrMh$I0o2?w-%5n_4?;nA=g6R~z_IY;}l3e~ZMoxLOni^MU+?-ound!Svwqox9!! ziSs+SyXsSfok8nF#Tih6y=rbNYi7Q2i%qQd^X`> zA|l7j2G+kJ`&3T+w$IFK%eyQ_EOdbvj^~q)OL5auG0Y-43A~3SVpcx{>Fjmcqj`uI zt?!k$UWGQL2J`34|4tq^mSEqn`fNzs8+1xalX3~-Li8zhB~SAmuc~n6SP1jq^e2AL zuJZlTUIlHa^o87q^-TE1e@bdnKs`OEzzuDIHdBg94tYrf{^^zlcX-!<{WTz^tDQH3 z_*W}*g6zvS-F8b~)_u-%O*kkaUMED?)f2fRwvJcJU`O)(my>^N;}>Pv_?23U@gK+O z$oLh*dE>RXC++I$Ys;g6h9q*E!WFu|p;H_7r<*Ia89@ZQWyoYN;HY0qFR*LfUpOK^ zl7O@!?@P6=H`JiMHYC^G&T&nrR%j+ewEp4BC3D4C~ITv;#gy_<Q6gCS5vEuf7Ey%Ke@7yC2W) zP_)yg4pBwk!zxADg4}oV4H=6l!MRxyZDpl90Fhkkugi-1i=0gQ0o}4-rP|H2`BFC~z}CauY~$ z0$BqJyNsaFlrTpDdgS(WR53Kl{|-=aJ|{L%8xR8qSXA7+`L zStgYb?Za0h-#c1jFQiOR9nA|;=d1!PJD)Mvrg%8ujX=IOk;tlE6|Z%L;y z!Z7MBXQ{WdgnEs3U3rll+Jr3ZWQ%O_kmz@HWC5SzYTE}5`Tv@r=Nt~TaQOvPliiAiSLxJ?k=q| zKJV=7#mkrTyK$b6H(!mYLO4w~>et<{I?#2Yt5*S0uFL7h+V*Q=9k1ak-NZtQaJ6h0 zKS!h)KYx|J?ck56B3*WqXnWOo>J4hw+!fxOu3F7)zbgFM<|^I9LW&levwFNayKI`f zVvJ4~=5TB4@L~R&|FbpPw{>`HxOVvP%HdA8{dB<1YJbTm@ulo=v9!wgJRg_w^5y(a z&hzp5VcBShlmFW=p5Q*+j>GNO2m)C>nmlYzUp@XB-L8h)SB;tfZ1SV_?27TLd~Q&C zji4mAng6W=;m;OV=_VFZlqO^~%>SlOtmfY?%QXL+V?qX=3Ts%m1Q%chCR_&{tznaO!(O>C}*Kbu^on^;KELK9Z6H?eQF ziG6Gy0E+u3KWu|y%BB?+88JAvu6j!2V-wmc_*KI(26&kPL3(gLh*)T`t{b2 z*<1U)oh;n0ldjFiW2$6ddC4?71Hy_fquzAG>WYq-rKDr8t}4zs-HpcVv9-6Qk;XTR zP^na4)nwW#DrWTQhgm~at*__s&J3SS&Y{E+!r)0>gg-e~=_VFZWSXCI3Q00)enia1 zX|&5`8f{BY=!q0^!1ClX=VsRRS^c1w$nO}0Gr|f(gJhO1_roe)D-M;kWV9IqY^86{ zEO=FA0!}vh34_1f#@w8>8~6H7-fK5HS(hgtu$#j6UDmzHPui`ilBAsU+t$LI+1rm4 zv-n9%AOaR?_D){uhN6wped7+%jQ8}gqJ==amLW+332-OP#tcSM@&eX88yGgh zU~ynTPuR9EO!8)AQ1qn2RIq?r#n>831&ayQrT>*u1})CpZqWKx(i7|W)1Cs+o?4&T zs+Ha_%LAsup!$`D3^qWYoB`jN|Hk>uGg zA4#5c03e}}eCe0TcCT_MY^sV9-G}-U*@u?74^(ftzQ}IeP(5cSyGe;%EsQ{fV>WW@ zIZnpcmX)gO*0%(asPR<=6;O6Fy9;z>YKGWDgWaZ!SdmkP2GUq{fC*L>Z_1{I%Cs-; zg=Et2iam54rc8NedWkx2j0pO?1ssC81BTY@$Pq+IVs^a4t0JwyAaC<&lkHgMv{^Hn zgH(y)X8)?6r=)g%J$()C)Rp8>CKO=^Bcp5Dfi9axM2%C^6*0YDWO{-=qw2MoUMgys zo{Eg=5h&(N50y;Q43_p_dPF^)SY4?g>lV}N!|ULD*<0-i^1+OCggt_ojU{?IbZ%xH zG=|JUOwN$CW0}6KwEC8cn!c&X`o`%s-Z%2xkO-S@adexy)vYYwL;rY@dRl<>at5ly z)~(_3dyJ<_%&r(;MkEl+A?Aqy=U>k2oBW_u%j;yi1^Px__20leTK{vougF@bh{V<> zFz(3@=@UGQ#AMwpFwKAsO?*MXhV3`z!iJ&PZ?68_I@R~+RdkEyrYoddzE%Fl%V@1G zXHBtlNaDT%Djt~ZD4FveVfN1oW+8SmwKU#W-F|8f_cNI#Pl21iLR#= zjn6&@+o3%|`@RW2@=mbK=D4u%WPf!QQ51do{|G%ZZS!g}SPw8`grEkH4h}jv8C4Tc zF4J^`=gPA-)7KnVx+81Te@g@&3ezA>+o*(#2C%ESa?vpe>R;DJz*$a=rOkJ39XhMq zcZDAprrSfLU2O?%@+KeOGHP>5W4Ymv;ULCpVv))tKJ}WXo4Avl&R*@2M z4eB5GBe1`IR5VZ(hb=;acnZ-4@f2fzq;hj=Abf$Sh#nhBv_W~FU%`XPmEgUp`uK=h zJKE9p9b)z$p6+K)1T(*WJipm59@17g+n>ilN8zXf`odS^O5V*eh{6YACT%|^>%NAY zAS~RegT`0FO`s#OZS|}oCBo$Y6hTz$4mo~QW9*~%wEeGfC2B!Gzv*O+sjeoUzCigI zE4uC`&@IyKe834^hgwfR~&lTxA##4~1AZH5TvR$UqgC^MjQ7Hf$# zQ>>vyTM8xNakgy(6ygRI;C8UZv6clqjM!8rbllN#g>?+h)%U)6b#T1UHKvU_h!DCS z+0L$&{0>C*pnew9>k3bQB+Au-sc5fD^gnhl z{xW`_%HDFApGZ8tHysA99nPbz?}z%aWNU7%HG9+gZMVF6@|>{4(6=x$M3c;6p{C3X z>SH%@_c@0P7Dj_b(vUOs7ktZu-Z_!R^7yVNP6`~ zwUv5m{kbeD)J&IA#VMN?gAb=7xQZWOGH7)r6*XN+VB)xQ?+RdY2yHopR^6X_hBy@rt_nDOBH%B#lbj%)O_gHH?&BBx>$ga~oAMuF$q$qipf?`J(Ma^XC>5Pt- z$5~(*U%LJ<3=~#%7(TTde-fjaV^LZvEoWvI^s@84@TvrG#NLg7yjo+mk#|Fj2`g+Y zVJ5H5q@@i@AUBpT%w>iy^Riq#a52~}!lYCdqo~ssY*4%jrpT{#a90Kl=bfF&`cZA^ ze1&EcNn%z1iFrX zv(ubC#x~@RKWM9d&Ldk2pzUHu*_8s=mD;D%M4|$yjrTAY7Be8XuW3|d3LwBuQ`DXs zt%P0NYMHAE*BRlH9}!7~Z!|ZvpQu|BeDPCEGHtD@yG*OBeJIx;{JiMB58lz^m z-EL^PtsHjST}aBI)(9?AVfC}MXzOf;wLaY6ucUG1^>0cA#&E8cw1&(^2$!>~#zGcour@6nEO)0zSkyg1n61odcL*RrB$xW|n5NbD!m ziluFn%Cuf-nWXGg0vnA7f&2LZv&%~3R|&<=ZI)dOqTQH@;#?DH?YN~W#0*);3@J6+ z9Ah29Kg0q0%0T)^<2XY_n{%AK_x)+;W{V&vsRqx_7Vm1{VUZk@NNYJ2mZwT>8U}Bv zOl{OEGq0hAtYDT=o;I=DnKiA-<`%VKX?k;9n$q!}27?2w({i8%JMd?7G+CZLg}$gh z$bkkCz@TRxXpGn#XkuWtX_L3KsRmjBMbt*yfz}HSv@@1{sBxI-9#ei9SLrgsCI^}x zn(5pJ)pDZgi8-Vk1hbB`GkL{dJN4MPFLS$M6An8QD|odN8Y6f%(!NOU=*kfi<7A&PyuP%*^nzX?fV(VVTM*jPZm-OS=<_kE`if=EQEBWIaz@rEYp_?Km`Fi-Ge7NYz?$eMc|PYxtgVFk{QQQA=s+) z0b_H7m^HmH;uuqB5p@EBWz)P%&B%)T64^zY^F_ANed%nY+h!ZKVQZFc%rOqzv;wMB zvkW96tFw=I2lI$1{V-h1N1R}qTkgdOh5&oR&N+r5Yn&|X=1%X@q5koE|3_2C$dc(y z@f_82n)Dn-O7t3&K*I9bdEd5oXa@W(9Dgy-2fXb%-kRQ|j4+0``hnkvgE!Sf@HTxD z--($ikkb;@?vUz;=*>Fl+^j2prEO;|MC@8787oUr>xz}-vV{eRb?4bx*NLLxUR@`O zhQC>}#xlFc*xC~VI(LH^*?GB|p>V~Hl+$<_gP_@Qr3M>=j*(;uD7PED!<<;b&d6=s zS@k@?YAX*216Mr=xYme5T6w^DYFcQWiQCK+^NXcnP?rpxHI;v zRTTb-F3U~wc4#9LIT2c`;GPx|D(p2238vUqb%6~`i-^<4C>nWr6j|vwP6aX+HvV)e zx}3k)Xj4{~v7=`*DFqSb#e7TxsW@?=z)tY0x0ZK_c_Rf+U2Ck{*NEcYlB*&QBb4bb zTpN1dRjyVpR!ROj%kf}`cD{`9Q_!@LvEX7OV;4BB4XVwH7r3>=YV@i*7aazC3$6&q zHxyjaOI(hD{}6o8AUmUITRsz}2=(-C{G~ zl(NmYk-TtyI4Bp;4SdUSyiz-=1YNc#UAFH8gmT|Pa?NqiGr>bQa7jN%*`pVD;cD!SCQ`3~S2F@*_NWSRgK(fypWq+ooTZwFQn5mlWp*mnf6ZRnV)y? zLw&|M(!}4zH)3wXhybUiX1Tf)j&o59lrG1)VI`Kqz!47M7+A?0L<`Ob3GsX}a zphYk)04$EgTgfJ)l-w+hsH3-bPafCmIalc>7KWlwKKt12^wsWvc)0g-lY_mm^QmX? zuN+AdBY5qy@%@^>w};=!N)CHMK!2AWb`zbWg=dL&G!%Kjc#GO~e-$PfQGlkx)FJ-_ zGO{A@bQsW$;`W zWtCYh&tp%Yi$(?Eqc5o6_uCZK-}!m|^%mR?A%-6bEc7@U8Jdw9H-bC0hGpET8B^jx zpn>`ujyeZTuxQ29Ho>ALOlh$N)Gim5K`0Q-X=?Q1613#S`6Z|@QWie(4Kt*`1qBbl zCdP+Z%dX7h+mlKcCN|7sPug3U_*8`4R7U=EkJv1vm!$&4W+9y{EOIzW$Dr8u{`O2~ z3zM%8fnk6{5}_EgS_dUp+33EqBk!;F_%lZeV*yrp`Umnb@!OGyDF<|J(@^=9^ z-jkLl9gU=-I4E#^37)wrXpsJK1%x5Bt?bktXmYoRA+VPn=a^hygqSfLJEBLJ$u%}E z$1K$>+cJs#PMMm?QaPBX$*^LI9XjeQAZ@6GMKt)hyb@IIQ&rCVTe8FRHF;5QQ#0y= z%Xe@He*vu3)ysD9k_ESjV#-FnY*fW2I`EOdR<4;JzNNzCuH>kdXzY@msLpN_qqfBu zaz@$^u<-Re>WXWE{7UQ?aubK#2l$v6NHn1I79a)Mgem@b+Bu7hdnxTt^E)mTDP`+v z`viN?s*6UuMXg=)Th~ga`^2B*PlYCo4^25NP9?CRpioTLRbrV8cxAf3(ph-np6W@G zZuhE|Gk~gnE|B_`TTm-JCYr@kC)gVlq+)^MzK5aj^J~ka(rWu{ z3s7H6CSz6h0rVCFzx!G{DNVC5(Z0S*J?b09*)DLp-8Wj>2vo*A#D)G8Rv<-DCBAu% zbmE^?fzAk@a`SYKGe-0>0&{#;QJwly2Lq3y_Le%>mpT|S6%gtWc0X)SEM%Ni?aS0b zK#5_~dTda3{t8HiG3^M;#_S8PZ7VzTM%}h9I1^qe$0wses)cBAQx`K|12VmU5#p`)w`DFR?X@AV-dO=E?^M*8FhV($LWFfcE(q0-T5c@}Q&F=ZRFN$R$@b_ALhrF^ zZT{K{WD0cAqKMmY3I=p%b9IN9>wR&7xUZpL`2MUG18*kr#< z`X&|w(F9h4V-;2}W!zSgw8*d5;1SyM{taPIc}x#sm4-PWNn0oDFY06znNHSEI+>_> z50wl%#uERS_GkEX!!9`f0r%3-ut2@naS(8qBgHY8Jq7(WbhhxPP$D-0f z=&MOyBbOsAPm?00)}d0x{Smq$^c&P4vW%?*4~t?F#A!P5Bo&)(|n(|8vi_&4Cu=WHGWXTUYk>?kM7LEvc z42u>dCDb|wm$mI)9{%*WN;k2PB3pz@atKW%NfdR>*UqMM?X0jn$rJ1#uge;FA_sZ> zs7s9GB)7Z^NRtehTF^D&gh6O1!;_Dv*>Ww32YAGA~U zI%Z%Ev*xO?8^FebBpN+zO)?MNSB3|6;D4bT;(5z7EH?}{yhvP|oC+k5IBbWf>HyW* zKNRaY)kX-jAY00yn)2m>HQVB_p`c0Cf4XM{f z&0)Jft-GjjFa7(af~~{O?O@q9zO|tUP|_rd)@;896}1rX&?jR zXdn|d-t48cexG)byi!VQk=!N?TW;lAl9(zMB7uXJ54ncyN)bf9?6S<*9ogFE`BKnb zJum%GZ-hMvaZd>v#zLV5MeNDR{=KY;uMmWYm$4l%*0Ej2qsf;DhJn=~=+3PyJs#Nl zA4u@KXhcM9TreWq9(RsbA_I{Q9MfXZ2G_61hlmh!s|815*_GK6wH}Q8GDstohES9Q zPhZ|AY@Hjvt~g5HIov8 z->#a~U{asmJff@z0Th|w+H%{PHKg&u)+9Q|8s%|!Ty~HD18vCeJ$!t44}Rh5%tO`S zq3KHfHGf~hia3XF#CKkb4pwediFu<3vM5Lp49uc%HV>R`u6!4*4UE8F0}R+;)3{J} zgEc+8_in-o6*~w1;?gmQFRZ%ikG$e=i*eA=k^9A*BMypt5IDKddHmPKm+}lS;Yu`D zc9m!ja7vct{K`_mHu)%w>@sS0JV8`h+75eR2(LbV4|)gac{ZPRkKbdt4FODH3E>cL z&C6Qc7b*cQ?R8>*qI*p;$Ts62zt_UIw0hGVDLWh0hMoCco6`?a4xY4EA5}*iLrLcy z$NwbfXkqg0#n{KWJ`m{mp12u zb~p&%8&|QSZopEXMmVR8(EJ;AIe!X?3K8fznWYK;^-L| zU>vat3Yp3WlX#C+WMLDm|3fifS+NCAsMr~k71PL=?-Q%CaFDGkm%zA5p1s`0<)&VR zgJ_N6;tAAUCZ04Q^FA(2N#HG~r^#?$)H;OJ3yhW8@m4Vcq>$;|mPdL=Huh*%FFY(M zxwUjQBm+B_+**Z)DK%~_^Le=?p2e3rysB~F$w4dw4mu+-gh|<5BVb(qZ4E^|Kj_M( zwQ6Y=f+OYGEah%)_=^m?H%Y&lq2VIZa1m%Y%BIo*=@BIJE*QC7&H?SN1Jel4pzp9Z zhYJE2*Na>(48mM4;QzLU0@ky=(OoaAxLyM4m>6fmgRdp5m9XzI3kt`wL?MM-9$IMv z2E!_My=bjTiQ!3fy;O#>$@9W3mCrj~NX%U^QXD%tEN%ci&<0vaFF0OwS`vHS(BYH_ zfFEW*`dS8ZyzJ9_>4~c_;Lo6q6ayhe^tG^)5MN9DK-eHJiPfvGEAgp;5XRH*GH9Kd zd9qsOc)@Bu>v%!C`fLvQ=^X4SH&5pkr*E6%MUw(o9mwgjVlpwO=HztI4Nez%ujrd? z-ZRMJbg>xDbtZW+24}g`MS0G7z2$WA94Jylsx)W*5}I3+J|(QKwfip03Z}WmTGLj& z>57mzd0ap<7}GFvB5vC#1nN!qIcG9_Vm2b)7$#XEvl3Cq{CGKCRi*naZ99k>QOA9F zYD%WYh>s`4gg_%YB^n5XgO?rS^D+*E+OoibEQTE%FxTCnCTo8{selwTZ-Os32#wXFME=KVrrYI<$2te^-i(PC{ zfjO0R{}delFHwRoB)5V;W=QDEIKkKHH?G7;J6me~L(GkfP8;Q);ff*z&q$JPa zIV2FmD{NindeC3)SY&hLTu?0XOe8=EMpju`tO2fPQ7P>Zm5Yx?hVp}!7NbO3@ga8J zRSZOyLZ$G+6pw6SRf;mciDei+H}d0A#^3nT>w3G_bt2Vu^GmPmHm~cJRM)M$))fb5 zp)jp6eg(%|v-(?%xz@s2yqXL(TJbyEW|>#ipY+nza7ol9M@)-v>s=0=wo9e6;gXnX zm9B?NO83)Pr-?#Z9?M;SZ9UZn;EnQ_5qjr%D?NLUCGTQ-c16$IyLc`LH4&N(@VF_9o ziCzfknggwbu4dEjlw}79*__IuYSSANW?%1g5~j_=^b9502^Bw5%Qg#|%DtxA(a@6! zFy#bmGfmqibAG7)4n^;8AEvdm=V(lC9T{{7y=0iI(1ZX>jmJ(pdt=T5Plv^_q zAcGDBs4YT<*84zVF!?NGX2Gou0qRL4Kna#FDFNJ56>4AFEqch=Rj;)=Q&BB;l`@fXlz~BOP#v<>*}(G<0h0u+6|pO94xTXyN1XMp zUmco-Ry$)-9(^g zb6ObZ#F|T9m|$E)o)J<1@zXmyxCH@)2SG%6)-Br7>j3Uo02>o?S!D-WiM3#P2kIXR zN$PrULW-X@c5*;ucYv&1WI2ID9#Cp77!Y*}&m|WRspbQ%nyW)I_44YWRCD*6=zXi^ zLss7bReeZ}a8QJIvU4YG(Ob{izO|ZJi^a=q5l*t$qSahl&6jH|huUMge10sKFCPmv zUv6U&QgX>yKt{GxgmI9yjH3?2h^jJ-r}K)9r~68b#~{_Snhy(7m$xBxcpg%RcZJkp zgVg1M)L}vDa>u8BQ*-}i)hv4Snk71x&4J3i_nH@LO=w-e`#+m36%xMq^VwTadVBQ_ zRK4>^i9C04s-j%_7{3!J#bv-|UAni9(jD7(waE+n-&sGKe=EX)>~DoO)AZh1e*@LU zGS>IKdfYDK&<$1$S}OL!n@}EtMz!(cQ!`)A)Q0Gp{mk05xG%(#3o4gQ0#6WJ)}wP? zsbrT)M5Ap*ilrW){T;NV6~)UGphKP6RgTMtS-eGp|y0mFj&nlT;}oWQgq3eEf%$KVR%KCFdMaE;N$dN zvVG_mNqzI$<(CPRW?*Er2dgt($7}(u{Hd}9Eap#*x>Lu$F%!yFu93cM{$`44Yp4HM z{A`}1sBkaqkZFW|TqbI>1cUXIPny)Qdzv|jK;k1znaaqULgrZw#SK`MWv(Q(lR3TQ zK9R@g672Y%nTZRDdc&M!d!~w@$Ld}SbW}kVU0|T1>_k$JWwY`rP=tu~m^hr%!oWCs zQ*J_r1-7|uOzy?hFa;Ln_RjXggNf#0WW_{r#KaQ@Pgw&f8+vSl%oSl%Klwb00Q0Oa z*t>)Ek6av>1@zC2q~m&+l~T~~8A}Na;@`&BN2oX}`mG`#4ES&UkEi{a-DyEWR@t1) z5TP^4>`t}}*0hbOirW?$wQx-#i;E^T)D=OU@IF@P9P8`q$2%k&?_}y25E+th_w3zl zM)@BV8=*oJ}qEIlGA3MTDvjbMtgYuWB#UV4N;b2yS?$r!|nF=`6S+Z%~4;5vmy&8D^ZFxG#fYAZ#vUr0)e$OxNXXNft`W8?}y zF8ci@{1{BB6@E+(KQTYCJG+dQdEGMPjF;T57dRMP32`7MBzr73u9fLkV|Wfc`|`Xm zDggO-x!nzROUt&Q&M4HG?hUC+5zkussY2NrL#aN@iL=i==Glf|G0)y-`%=-(7>RYb zxz>9Aq}``VHz<@AAWdpD*`P>$k;Bg+N)vubIoTyNTGPG|5dD+Gl*+3U9p?&)F0QnU z#PFLCiLrYsuZwk?&xL*zb#)Scwj_aO*I_vKmCe|1}4Lv5W&5Qi$i*xIJ zA^c-`QlTIV$Qf#m##9O3*{(xjif#08_d?k|xohE-9KDn@;qECsBZj;8sFTj%LDVW@ z8134_mR&oDnqkYXo!oG2zw)FP5Wlbw`@j5Mws2%t7MBkybiF6oNJ^vfAY9`qf0#3{Z5hKK`sMz`;wT`LNU;dxVB9X9gaChWOc=R$g5Y=em?l5u<#1xI7UJz^=j}f$tPPVi$ErVdw7|h&Lu<1+4;uHhi zT^+F`HUMft7m?CIG>>z^iVZwb^gY;L+(8nUWyEFJm7Utz3G8b3Q#+Mr&10SMzXDj> zuCW;FOzzC1AnrNPH)O+>W(2>7QrfZ3CLKH*ST&3LyCj$q=Cbpu73Lfks9Fw8J7)EE z;xye<+Np8?J_gx}P7==)0}m4dyI@sIXGm7Ov5&mizL%9^P0q$M5aybdQ~ivsmA5_ zp_uT2!)^C-bO{w-hzXYllgN|>uE^91j7%-KHf}_*qdy~=mfC<2$+Vm!(^7j73Zb$( zs2a5o!Bl;;CTqg*Jd&v&qZ`RIoG=74yRr8(QEG!cpZl2B<} z6uNv2#04Nv=P@Kq18)Cqg~5dtr^SYqF2jXZF0!?(K%kHmPT^+!+`>1v`6B z+?e*D{@WZKmgSr*?>k?OgJB;Cjhtn`4QFf0jk7HpSG?)ecu>A%Mp`_{MGOPEvC1J; z^}Psy|CWS-D|~7i)j$$O$5x?M$*a90&vHgu3QJ<(heBPAvRUsue{Lt+9?8n#@RR+o z>F1vP*YQ}hHf8mnCTNM>JnRC~S#Ky$=2!n_c?c<)X%-J;(`++0v&}Z;mk@01NoTv- zNNH;gWDYQ!%}%CM3?`4br48Vg4qUy>it|F2@rFz~p(KSqj>9EucEv`9&_~5a>XK5$ zM!vvaIR!>~U@n1?HhbIqg%R^f(p+~Ct10>4r#;~0`O|p(MFCxA5Ovr8wmLoXn8#{_ zRY5zTN9Rn866y37u$22Ii-##915foei$ zF7yqPW-v_F}=ZjnA_Tx=S|4Tr{5-w6|Roy4o}2k!8`?9`Wfw1T`J3traP` zuX6cj{>)=92f$N1$OCw|l=7Lt62etI)KcksH6h-~3$|awmfY8(b`4QwEr8eCQ7^Kx z@&B6-SNmZSrzJ2!?9;4i4ZZmnF~Yy-){}oOO0%@b-i3L_3(YaxExf_zb2?W;k7&gV zB?EUBl19U8!z@O{mbSoVAGO;jwIE%sM2IqQ?yJ^IaB6A(dkulcBm{_6l|f#@d$CvcOxn!5nEk(HwH3Fnc?!5 z=?L#eH!RI9-mg?XEkc?|Kuh10&)Af8ra{4(Y)aKcXP9O0z`YQ7(9)6ft!$HehpVZU zP^%f$sBQMyxKZ?0kFE{SYocp!^R2SMDC)xCHfmA~e{y7CbhR@jgbo99Wb6uNzC>Bs z$-Vu%t4{END26TgK;j(N)eRum$_;mjNkqIb6tFQM`F_qLkOMa<#YSKmRq1R5TvNy+ z<89@)u{-e$K-lmM&u_zYGX6cXN$xG9yQ1KiMq;!Kwz`|85|ZvoLY6{2E}qe~eo{>nS1P0F36FMylZXt7aBz3+|e7(Opy8s!Mz#um@jwwNf91LHgPYLbvr7TlJ!e zWc0+9$&|bq*IkdW?#lfQU{N>#7EQ^Xx1tfkufs;*ED&AwNz5q@H~sr~ZU@CYtl(Ns zh$nA?RssO7QT^?(7D^}&gMl;qKCB0zW$y&bzivAVaJOw!?#iYdP8GwZ7|tofI-?_; z;DpOIq+8Y?H`3U#PmZVLrOtT0wISV>8&f|lTkd)gH`MR1GF#1gX3^LwWSvO_OzfLy ze)3c6RvDq$$*t$u)LvUY08OFv%czWhPgk4@QOMo^E675o1zyv*$vXLf<~}Te@qo`%ux-Sd8P+w6%cT^Atl47pGTKwx!=(F)fAm8Vv!qTW7bm$^Fy518E#C zO*!`i=6RR&p-LDIEKtsOJMz(TbqHJ&!0t3KH8k`G*m&1@;N&9bc}Ate0H!^>=J?=f z(}4bNAEJ^*JLvD?%IHef@8sO^`v7IOpw>ADBx2PN$?#~GbbC{5z~!TBR;`UvT~?Kg z@zFvjNLC0XbVJ~pi`Qcs^{Dx)8pC!eFRZ$2U$+^qdLlnGP=73Edl_6PTt@iTZ@6K4ka;;EweFzzIp03WoAodKDUyEskoPg4 z9;|h!`D-}gBj4)}&wRRJf*q)}#oBE$&>ueJTR*xQ6z(unXzY@!M9d@QK!n z!fL&h?oih)RTl@Zv=@@Ug(KLZUTN4g4JVWS?oIm#w@sB%N*}bFo^Js5=igSv?Mktl zGjv2uiff?So2P|Tc&<)Jg-^8w6<%-XWbVS_8dQPoIXA->1r+!-owpa3j387OHg0hs zFS0G1y+m?LZsCXrl0M)^3rMwRh_H>j+7WqTM2k^U*iMWIF?mYmd{94kZEJgxmE663 znJQCt;!&3av1u24jQCQPdCq_He+;4d$eE6&M0}vA_}(KOa$QDy!S%jswug!V!`$Em zdAOm?!=W2p(dHq*d|0%3DAc)q#Vl>As_7LMLz`FZO`BI3Z5~Rrc_`54p$nr;8?4hN zmvgjf4>fM1&BHH;HaUa*GA+Ur4swOKnjyr4B1BFpzuk}4JqS8?L+G1*mT)aPMJLsz z!7vw)ZFD+l=oHwz3npk{ZF>@P;`YdF$A9yGdSD3O27{E4#O|EUbJ;?yh)TB4!?Eh} zHY!~Pv@RDaT?Q(tmhH=^PjA@W^fE`HoL$L#{oy%0@-joBY@QnO9J_Ut941Ps{W8O- z%fjF<3o`Dqwe7Xo4u+>9-NZr_*>IhmuZF=p=SNn|V3&CP?I{UkG!6v z)91O*rNsz3rYK;8wK1z!ETfp!qAfl$t9%ia5P3T2@w|bjj%DIVGTaeQ6VWJ_ZQkyL zMwXyBqS3B_r+c$h@+W%WX>Pn#V(4xrL=1KpPLJG6fu0!nVBogr(;%d}B&C?3Yc~|ffmNWKFP1C6++p2JHa%1k z{lk`iY@KDWbvm9JZh4rI*amOC-PNfjzz8p(LFudAU$}G67uu_7p;RrWez2xsP6BV1 zMgM}A3+5#8h2v)6#V;v45-OnVNGxn*N9wvA2|d*gSF0%J%1j%q`-MqNgyTpd&Oq32 zm>Z#q6*kl1MYAvJ@}uJT9U2vz8>%JOkUSWl9q1-=JBB zERoTu-lU0C=u%7y32ufHq&z94rHKruOHKC+UJB-tW*YKB+AXzjR{OOuFXRTl5i>NI z|E^}xOc18Z8yGgmFJh*;Ax)%`M_yo>NaC(;RGQJ3mL`&~ua>FS(?rI^ke()TGja|G zu$Ce^W9f;!vRU%;x%NmC>C&x96&a&FTd5+k*tb(fk|5HN5x=CRiX@}Ol2nm^64q#( z&`v#b{%6S)XDxGXBNW%HPV!G=nHJhE|DSiY;^HbBIq7iW#`JSc~)4Iq`*?0_7QL|@LMYh>K<=}-|3rJXhR)}_* zRO=q?1RTyXMHUlq0HHSBug&(92nEf(O`;CG_KjbCCO60>>)XV-g^Nqw8u0g(+8#1f1Lgp{aW_&ByS3wccJ*PF6eo8WudVm_Hwpy&yHv_h6Tib9gA26$EBL#OBc26M=5FQ`F9<2Q>zpUbVi% zX;Xp4g5m0fpD7k>FE%rKK00u3!Zk^C(2527AnOrjCr;#)i=2u`PQk`fw8$wHHRPls zBPT?-$N1VVC!<;16G|><{s>5ZCfUcbvncbBT#$ED@NN@e`rW;vxMz58;uaEM;tU8d z<;2&q2Q6QR(&jUhxP&AVLUbrz84q`fu1IgFrD>u=ELk+5p=H}+xXuXvwVJh;4hB6m z9S&8LBC{$U&p~}_oL-{KQ&W8cT30en5)nRAc0G1edw+XvRPC!*WB^;ej<}K`dvf^y z&4&qNL9c1uHWh3`E87$2tNUyvxatY&F{fC7!D?6-uA1e9zznTZ+=9SmvogsQoM8`T zK`7*`9#@m=Fy7|c^F&Ms_gt!JPW)uI2)imP;svG(_u&6GALcEYeV2GK>wl-Cq+I#- z*h^{SevVO8!xfMwOlPWsb113T6d-b1Cm5N;k-Fd?=m-Hv;5k#3CUVhK#X3Rrp}DDw z;HeFjgHGxd*wm-JSidxj289GDz2q78QK**2jy2`R3V|oKI<}}ptPWvo#j)AQlHE|x zdJJxj==``Lho72uJ5aB3jOWp-_$2wo3)d6{6bFpZQn|T`W5gEQO`UNidkr&7XJ40n zIM)#nAaP6$5x+j)uRgl3Sli0-qU>~g{lUs`G+tfXH(B4mv3cO&<%Wke@Tr_mWRrh- zdS?gv0dLFk<)SHT_3>OTzdI-C`X{aHsMi*F+d=y_ zr=i#Yx;ef`<&K6WTn+sPtjeX28xJ5g*|p^ncUQJPKqU*}09Z*dwcb7k?$aqWHu*OR z)BY4luep3Ox25ir+{*nr6&H|pO%2p{4Mkjc>f-u00AR*zeR332uXV+_vg)%$%YW2q zcSa{Qu@y9+EMP(93Vu`885)Fv?=sLuFqnKuFa@4lUXZ1NLlA~0cM0tMk#(93E>@ov zm5iXi2iFvlTvX4n&J^|C7V3INB%`^}l(;aTve*Z(0mc@)EMD(J6VVV~;@Q(-so$$^ zVqH;4J^jAoB=Xp<#_4J^Ug;6cr)K5FyOOU*rwsfEMa-i`cNs)%qL~PEKsATc;#w9Y zK7Zi_O{^Z)!gNEKEA{h+tm1X7DU&hTZTOVP%xv=00h9-Khnzo}f!sl@ijJCQqR|^b z)Lq8!r-~tl0#K&bs{)n=#}g*%9bduw2L>>Gd=%jEQx(T5O$vbPi6lxX>yAPehu}B_ zl#dz?Ek#a}f2bksRQgeEhNR^ZB&AV+b<~D`qdGy(gMR~%gjX<0&B-D3jzBhL&HuW;GhW@`&<*QbQ)agh^5he=QrSrv)<**%(Ul-oS>WI zjp~)gMYRocBaqXWTfm#h;jdd=4=fSBzHVG&+Nsyyf{hW7)v(h$>z&i9*dlWnA1cOu zMdA=}qnFJg;x-B=w=mTXf%9_;@q%lL7MFnIO~y7ki60U+XZ%paw5NzdXb576D6y$! zA}a1jSrAe49|#efm*+&3p<+aeGf=GHf4XofxA5e#UdiqyReA+hX(b%umi+iD)fwrVgC&>7B+?yHUX%nFlf#&vU=Sl4$GQZ zhIUE0lG&K7KgN+1==M^%Q3T^fPcY+LI>A6O$tpWfd^sKaqnYVb5z`%MnTn*;FP{_x z{jM@3EDx?*Wpca;U0TI||KwUX%kx}&ce0wJ30o?t6#XEXT?H+T;l?6Uu+eGKRQj$? z5Vp}Nk|u$PW=^MAu7ZjWw}i{fP(jgICb)}4FyhA-$++#1#wGOHo=!WTXZdvcqnYW% zpq1&Itv!}q$m{dSG>_pFQ2g&^K6x~78ktXw)5HSzR8Bq|o?C?3 z@}c@88){tfn7xAJQ7v35=W*#PZ#lyTjQ{V~KfaUI>-8Vl@6%cR^V#HM0uc-TU;7O` zn>5cpk)AQ}Cf!hXnTe_9_$!^Vv@%0)y3$g1RORSrMx63kM!(rpSWC+IEJTe%{>8*N z+A=)S{tG8nmit{(yuHZ2!1K#0NscIts4UkCc%htaO9hl(GY5IMrB4@8{3Kq;_lC4I zhJg(qrPr=C`Zp7DLTC8K)jT}hdr&QFD)}Yd?0|!;Wh#-kGO)`9itRRw2tGM#))i@@ z=DJKD%~9@dV-7>vYveT>Wfae;1+%$PLmMT_%#@YxlkZ3x18$)}E(T;yMPfk7} zWDm3B5%lxw2rCbGi9?pPDU}GYbr(pgLSx0Y`n?bx+5+?3W>d2=fS=5)55yS`Il+PU6*?5ecVdm|&)gqKk6Lchc@F^lHqIYv=K?UL|s$TlU{ zg+q3bC|XFVU0i`VcScVf>g#yk^4`upN~o2NGOBK$cKoGXd3MN46OhoLABv2ZlG@nV zrxU%5ofo{g2c!*4O?c|c{TxemwZ8&iu{}qYYM&ehwVlS=JfLmE)K^fYQ1JXVXpLz* z2l?7!B@bc+a9c{A+?`-x85Bq0!gLH@=i-Q+{dXt#~IZg1-Iu|S|6_aNz|+{=24J!_`7>kER~EH2Utu4AhM!ex0c zF1SIM=}d!@JsZ?iMQ|vwXfG0(JdXZ1K6Lk-rTZix0Id~>~@UY^G*fO`j+Bks=Y`=mS^^@Q24hxG4 zSeG1FiIr6AFAUM7Y5Jg-uWP%)K`Z2~{SRTZ_AO+Lb7&oH#;LK-m93D{fu+b$`oXX| zKzGfcGL}%rpDfzv@K}%3dFBfy0k4+$5%aN7lq39X;40u})8*lN{Way>OOb+ddtSG_ z4wx;Kg1~K<51lr|T_83!s9wG1=ehVJ?nym3&dVZ3QD?T#yIhLpR9!HWR0}wuo~=5+ zo{b*5HA1zA)o}8&($b`e@c~EDYx@EtiY_O1Z<_IG#6+9u%F*$rqq*hzmF(Ht!$%jsG$=N(W#bGIN)Yozs;{;QZu3J1`HCNE z-vR+OMw^Z03?%R_A#BbF+e|MqW=@S!8&a?N3C#3Hm}yoM7lz!<(fp=a)B;-?VN?blbdIe?!KQ4HSx4*rj6|Uy6>|;m%7t=DXj7pOroe=!(s+Y*OSq1;D?i!egU&Oi~1QcY}2!Q(Af7C<*b^_6RSPjAN z7`B$sQaq&;d4uYh?g;M5s>$N=L#%jMg)<-*7AKNSzbhSS@p)s4QQ%56#hBq`yA!2Sg;|Bq*(>PLXLQ2ur1LRq*A<)fJb5;F1D^Xp z!QzhJ_XP9E{pSCA!2W$t+Y9zUA<+{fGRR3O<&ddAEsx{01 z^?V+f#Z3?iEF5b_X{nu+TKC9VP^I z-9(Ak(sZ)jE13X7Jb;Gj!%`nvXV3JJBhTH#TMCtU8sQ!FT{>7g7ml6Fa{job9_Dgi z;rqR7!mK7g`yGgUEy@*((OBIh5gJ5kqc*mm3?RV$b#LIMH+^H>dEafOFkw8~kO79UvV4Vf<39OQy4s547l zs+JMC#hg&+OyQBk!x+Ck1FvplG!GQTZfkXOF&BY%fCi4eHo#(T73!nY$c`|{4dw`4 zFgOV{6B;>`v|ew%g07aMldCv;BDrQuhP`G~Qy3ZaQoq6N3^bhe_QZO;AfrDuYQ%ET=p^9Z}Ed~f&Dz+v)Dbe;<*_zneZ~nnWv9(`)@MW^KUl3bUgmEW&%&y*5 zOLHAXx>~VZKkEms5Ve|cFo8V#U91Y26HSZwvQX{jHKz+x@A?o%TfO59I#->N&eRp* zU?W6l0;9!WI`zm?mdrQVR@O|ep1*cf$XGjPep9=v!kroz_v{#I@mHL`;$FpW>VzpS zQKr7qbcg-pnqo65axy!(`JY}U-MS44AE3oqOVMt}Sx48YmN&+rOmj*)#-f6;i*{j4 zr#-cc59(;e8SR2^cGoWMSGE`5|FEleVdhc>chPDlks3?+G4KoVJaQ*FKz0e`Zh-X_ zqGs+n{lX=wXIID7W@q%p;VaL^zKztv-M_j_Z1t6AVnDb96N80a0#-S@cy(6XkI{#S zpsJ6$cg~{^vzmkCu)`@LVJZ%vf+b2;F6=rB%u?$G;`Ld6v4%)Ss6vcPoGPwy+w0!6 zW{n1?W~I9oR=TuiXFkM5D$7=jHZwNm>tHHx+^V*Q^M}Fy6gNA-X(umzJqrxZ%V?H! zUJE~h2Q@YS5ni_YeQ;E69IIgmYf7TpIrSWu^~nz1LQDkmRQ9 zEaATobc!x&WJp$k$G-4|b5I$V#aYex?BfQ9;6}WZk)~$TK_O$C0vT;lnQAfeEO+Zw zj1AmU0LYvr5Ol_g8H|P^eZDu+A1pI>$$@jR<}NtNK(-EWrw!bxb?DUS@ zuIfAu$N@pOx|RH_(-F>R`owNgKwoJoV7oM~*v@Qm=BvA1W2$HXPy-&#gR-d!ELBgg zdk9THB-V@{X{4oIrQCja&wLKc0<~rJi=NX8_=R(wN9r{AHTR(3Vf6n!C32i5%8py5mH4 z1wAn@c9`oCi3R%AAZ+rX_)0s6OnxWOR!NbC%PNoPEUWCEgZ%Kb7d%J~lpr@lGLvR| zE#rjwKtnj+_(akMjI?NuJi1E+Y+!(sHSpSM_z>CZsfYAG!5w7$k)1liOk+l%uWD4h#URGon2*135L|E@ zjmwEPm_Ora-Udfi+4R7fybYl9U;!g!JPvgHP)CZh!Fl-J0y5>5v&nJjQ=s%8uu0|d z_x{_|7zP;J0UA5@KPr|>@I5a@%#Cq5lqIji3T(>XJe1X+w?O@`tzXdZ!@I85< z{r*9f!W!4T(nH(HM&&`da8)?F^7<=;1D#(PtZ*$6>{kEDyXnRUk5~T?%|*=rgQulk zbn9=Y4lCz+(@HBo(mP|j+o~Vd(KFnmw%_+3TK(CC!9hX#2UpzWSJcbVL+h-!^LzRM zr}fYeM0s6yE99#CGxqO&?D9vUK__^hDy^sW$LyCK3-8P0js?vJsU52PgfMwu;QFfM z{=9LGI^l@8$wv^Hm=?Dy=7Lsfw5-ejL0KOrKX;lwLaFt8)fVn>f1*=oj@|z(w)~#<6Bjr~qR+(xNR=5CX;Tv<}>amBp*^J-jhsII}*~rGkM|WCE ziSk&{nl-;RxP4IvS?W)wQ{I_PhIOyUMUZb>p5LGH#T}!!Xe6w8>a}LbH(&~P+!cEt z+3&$0s_;3LK)2b~r0X@j`g}?MnTz5WxIs^E$Ir>b`}uY;Pj*$Vb+pZ_9a;ncr{y`g z)xB??4jOzzpHh;k>!*aYajkSXn)trwz%dXS31;;#KLiSx`Om|v0G(&mLj?r$IocM9 z0rW{?PYCPRY?u7@5Og&iiG6M4El4zS75yF6 z$8KCDH-$M|d4)h2`{f@eKCvhC#aL2a-u73R#2Tz?N`AbI-1KYitk14Sh zr|+5#v21_QLw_vlvh`khyp-k5DFk&}X@ABiR>u{H=~MgN3y`qmrkH+{E+W+Hbe?^{ zi24`Z&KgmHirSE)qXswd1VG6=Q(Vs;Pv)y0+$pP!N^UqgU7>_A@_L^Ww8}e}%AH4q z61smB3=*M*x%%Ee-fOO$d}y^JMV-KilT=M&hQXakdAs>mu#3}QfaRyf+UUU$JJR#t za0ef~*uUdj>zy4`?ou-*w&Scxv41VTGNwBy$%G2k-?z{mXE=sA4V?bvZ|~J>Yvnq! zlqR@-;psi!Z_l#`2&_j#<{Yx@ElStTd7Sl&`|NV4%NNVHK$*rwUwrZBGtO?Ibm!R@ z!7zg}JkO4Ff!uYZ%gz%SfjlbP*}3~9|D4OF}! z&{7%93(N9zr;2w#5Y~iT?tf=}>ZI&{b1&)H_?=K+y_0|IF3)>=JTJqi9?dmdc?z;< z`lRNl?%dM0T6MupL(iUC`f8|W*KRjeSw%K5WVa`ab%B<9R=9aKZmNBQqQAE%>BG)o z@y0IDxWnkn1b;KQLQ$bz0pF9<-X)0>RjfK2{d!@FchIIL=|Gm?gtFOYZg+yfLGZX< z5LJxR%2EV>wH-VBI6)(81VtCCe2$#zaziOKN^Y&QLUKw@O2mNCTQy*nT&9v_hdo@6nD22@D`+J`E-#+`CoRg*{of|^WyWc<0zu*7g@A*B?GjylUvRgc_ z`D}sDwmm?!gb+Zy9t&qaFuPFXnTyjed~tcUcbekx@ej`j*^9G9U6c2{jaqW}(hqHS zI&2+=Pq^V+YwgZ!gI0j~4LYGCncHlwrj6b^;;@--N(EpW?uGhqW3cYqJWf-3d0-2+UYIMoQ)gGwHl-2P2Ba5AEb z!yywqyE5ozFV${q{ux_7>N_gZWBg(FDL%R$4@3Wgpq_itmLznq81_DOh?-fTj9_}r zBr25%9~;kt))1pp0S+5oi&AvdR*JRdYMRs0N;~TfQ1(YP^R7|6(_OyOjMDwAk$gaT za|lhhHXv)jSz>bhmazeE{}-62oeQGdcV9Ol0EK8eUYG9WJ_!B;lp666@*PT0HaN9m1Ey!e__<*bBj zg*8D-DT*=+BH){ROrMaa-2)atgDOL{S5);WtE%a$5^&2`e_CxtQ$uAlB!rD`27>Sv zr0Of_XN@^=k$HUm++D6F+vk}|6jDe|_)%yHy#hV5keIy@(6Ya_Zia3I3=vRl zF#>iq+isESQvnLw5`f5erH*4~tz!h769x|jo+9z+6>5voZY4Lsz$~IOKodLi$8m`m zWG)6VUkDXq!zKo)TcyDh7Z4c)`kA%2nYNJ!L1&KOKSm!kzic~e0AC?BhgAjr!w}GtBROhE^IbzDrlJs;DuC>=4;zw0EJYD zW-iKw=Ta`9TD)8sB0UeFq_v(QdBV#Dn8)}6`8|~2$-erQkIdr`J2m6U)*MkMPzXOh zUKC|Uh=eO+^O|JYOuzx&mxHmEDD*y zA^B*+Px~!Rr`^L&{`-cOvJC>&;6f-6UmlCa93#g>b&f*4u7&$bA{~9iN6vCYVS3Av$oPom$UAR2@=kjh6qX~q zxs;Y8>j(3jEOh!}v6zAgdha9wQ<;QP3%+ z?kll>t4%N48f_J=x#`H4X${E-XI&Ihfyg~JSAmUt`b_ZsH0FuoVWYm%dM!s#ud@UI zL8#=vYV^2bV+Ia+=3SxdUZUG5ic~etv9O#yTQmmxYiH13M9l?@nvYCHlciIoC2Zkl zbLlV#6N~||CiE)KXT69KR5$rgl8ZSzNi49(f+lW5CRcD7c<@@{nOtttlAP)2RT!IG zv?iF#)jA+Mptt*HsW%lk9dsgi?Nof1NZcmoCKv>szZuU05ODIt&qvPbOcKPkk-Krr&KEEkHQ9EIST`Tq z{c4>Brn;;*i6q4)Y&^VGB{5c;DXfM8_?9`-ViSs0qA>{z;5zv4+pn=o+t>&)^iZz( z_$D=_MSCdI>`9YHpYQ<&x(cD%Irq#!I56OBrR{Cz16d`*iOYjv_Rt@E@bN$WjbHk` zlu!}aVA8i&PP*9W@_ z`tW9aBqA;$^>vDY*>3lc5|dVRp4tTvR;~K>402e$u#x_$nr*|0si+neBF zd7d`Fe%gbys7OnDbrJjmU9h$sA@w}){3NM`0Q!1w!0MI2qAntwQxWkX53C6TRx51f zz*000IU&OeV6}W%u-2la0H)E%>!y{6p9IS)K!Zc8KvRs*Mkh8!Y&xKIQi#3f&T&th zu0l_zzV2Ejh=v?Q9Rm>yumvJY;@va%rs)qMs0$@^i9UGNU^o8k-VtQ*rGTfe2<38t z^tuJS2iOE{t_bVgHMbhk7f?G}soVTA~_ z`%FlPTffcXa|>=EaCl#{TqM6~npVVy^Jr2(BP_!C4J6A!A?|k{1w%^CW~a;=Oj8ok zN#5^!OWI$a#Xx5WWFsP;u89ae)y|2>Yfr%>^*6F79A2)AMOz-J{&HWnX* z*(3I3Q{~BL?a50jPyWfCytwk@=T4{|4oa3`YBm3gJ>l5p;>m9=dGd%onW}vA=_OB2 z*pq6_PcHf9pO-w@_c`^a+WIfqljz$FpO$dSs^fa6J*oER0ef;;rPNV-;&IV?S1n7cp73tiA0puUFfIg|{Lunn&*KqA zCKf(13-w&~l?hoeWOs_fp1zAB%Hv7{rEr;JKvlRK+1|fuF&#+$*o~8$(%ojaju zc7-z-m{dW7JI~3iW6+;A+=Mj-hZ~HU6E-IEbQFT-t1$VX2v20koY)^4)GWk61PURBkz!WO)cC zwkhqW1qx;4^(V2Wt_&t^d2U{H^-8TWkN@@D)axQhzW86G__c~3v3=9zZ_9t(=GqCm zA-JJmHX-49_Wv4^B1=R{!Bz4Z|G>7HXOfA?rLB>ZVwvNXqfr*JIK$ZrT~%+$-1@VX>D0^E-auz7)329N2I5%~F;#$IXe_~^6n1{tUl=rMfJWO1=jy7@hm}AVs9an`< zNy4VwGHl&`iQC&htOP%vMLhd<|4jF>yH}Ch@mzzOq(ZXb`cP`Ld40%!i3Ouo<$!fa zG{yXM--Ue)eRhBz&!);4NmVFN!1C;y**xbgl6FV#*hAzo(g=@O)#aa^#>B@VARu7$ zzzQuQ@pO!Bu^pcWeVyO|j^t$<81bSd%{U*!1eisSmJ?v=HzKUoZ%%?Sc}8rhZIbIa zC-F9ERnzT5`6Sj1OP^db?Xx%I5e>5va8hLn~_) z#6W0;I_s3c?Gj4hS&b67uTcU%bd3_)tE7bQ7Ws&c(GS2AI#8zSVvg4{Ut?&@|0#4Q zE`odDr(xF*nJ5E33{;%_jpJor4irce8{&U;v;S(0tVC7lrOe>NvsP;0RX!6jM3Spw zQhUWuZQ>2|qE21%&cL%#AeoS@T8Dz7LvpG{IwY&WX$>r^yb8j$iagMXncJ~TA~+Rq z6PhLh;aZv4QSHR-Y??t}mvgJnrC~G~J3il$o$?8e43x@Ccu1vJGA~h4d|ye`EC6KL z56E9T>>c(R!)G>IPxeBspREEwBodDTNOP_NkjTc9fChM;Z3> zClGyLjI`P0Lz2qItv#A&#e>MBo?McF43P z&m0%mlyPvrl=1c9IAwgSf$bk5nbU}nA^mMgBD#O6%e72n+!#GU6-Y?N1X$DXdoc4| ztW%P10{afFfo{s?-?E@6C`bGe@NQ*qyXXF6!JD?FEmGL;ddmU|$SyamT#xi%7H?-%$ET@);{$u#mRtdgz9i3^yTe4 zx*O6sV~H%D=4L#^EI6SeBD%c|qYiIOzoYCk)oz|2+mSfhIC4Edm0JvMsbTtX(=uIw zi>yyK|JtwnpRy$)k6#>J+U>FT9`Ijdjnpf zE9Gdg7Uwm5*%~041`%i=?J5|?s!efm)ker(#Dnq{HmH}>8s-{ZiJaHkVzf^7o=5im zYy_iq>clL^%NKCu-^q?Y^ySYt$A1ZLkhP-IMljiFQN+TDu5s-78Wp21=558JZJ}Sq z{`B6rvGfau^wp9u%7gIt)mASEz$^vz5i=)~<{swHI7aoC;Ry*KU7YH<2*xOeuAKfS4wzPLmV3Yo@Uspcvj(N9}C1- z{Bwab=Zk1&3Cn08VwT9gtFntIo1&nTj;LJ%`7$oFQ+eTOn3C79>5!>W8?C`XOr@3) zuZ|l54_3WgGvhAU$oaO2Jk_iXH1g_QC?&)XfsnXNLIWfzBMhy^Xngpj7E1}v@;~>n z{^rHp??~Yj`Z9j71pHX9XPw|@;f`3OY|J>!6_lGOD);HM3?AzbDN=PwB~&Udse}wb zEL>FG$L`+T(GNQxBs6c-1rM#mRTw%qGz|+Ult_S_4`C>$j5<9SH8FNW#|Wae`Y?^N zCrpRr%;0Nee3F~6N>#&9Q4y;MdOuQz-A2O+MXZ9DxS|zeB6uq0KbE@t&;?-y%fCz~Wn;H?*)H z3ngti4%0}P@wc=FbV1Hx9nWE~T5c`pyarI=wm(~di%AN}TQ(@Tm>)tk2Md?IkX`6kCYb9&VOM+QC@@-yW^!ENK?@}@HnWA{ za04bQaf~&rQxH&5o5iH8Azp2BHPB~Dk=!s+2O1>S0fieJ326L}Y5e+(TzSoO^Gho% zEz>}2Xz!`foqY)nH=uJFDEe-dHIh|=;-|*@R<04TXJeGFt`X&c2ST0U`GmfNC6twW ze0Ca@_}3d=$XJ`Y^}!s0yh1RfF=#9cHJ)|jd*8_T%mSjggPJTaC$;L&d2O-foUeQD z2uUl|^iCNm9AOekm?D=nV3Gz_!9kJ#g?;*+aYs=sC5d9wBH`R~Hl()I8O3u><9sB2&PhBkA0i8(NP9;U3?UOVk z$F$MMtU=9$@Rf{5tEZ9uRlSNa6==J&jb#P$8~h{!2G8eItyX=gRt{1zRlD&vHU&3` zkTVz}WYhx#W%id#n@cMFO|&~rtvMujSZPOAsw#yyf6S-ARBEpcM%~nKR;(YrP|Ep6 z?_K?{TM?OF3`;7M|7^lfOUg7su8@%iC)t?&i4ATfDc71;S`2B$qr(dnN>Wib3#OVK zYLp6*J!;j}mX+D_PgR={HbAByCIcSd4$Ne$XhOe)#tvO@J%=J?&*@LOxU8&_3ddIo z<%d*StSoapCrK*9+DGxRb-5|VI2$OqB|8^xB7HB+bs$(S!V!+LQ+ zcE0KAokpP!laIuxvZ%I#ti^C~iTI?$FWYuO$!x4vkjv%Pjasue1 zPM9K^E%4~`m4;qa<#sJHz?H;Cr&vjbJH?R~R77Vv?}8t0WuvF%8;M(#oOQfP0>u}R z4Um;+Y3Kz}V4fSu(yVX{ej{a08NB7D0PnO*!vYmB5K_hgo?K%Gc$pKDV}9M90N*MB zPu)!_wWoQN^k{U^o+!-d%ap<@!4B@X4LT0&iukM<@|E>_>XHE7^l(;2mxn!rleBf> zYm98(Sdq<~*{S4!Vm++c;ssU-#E;-lQXrR%6g{dfz~7@j7;(r9W8^jZ*bESMRL}x zwL@qXVv}khWLH$zT&wa{QX#NYbuvQ)d6PU1@d<3L{<*a@3V+y|YLnyHmyP}Wx-FF! z8BoEgDqHf@j1NMgbk;`Sh70uztJ42!AVy5#Y0ap@GQ=vCGTDU^v2N56>k4V=?lNf zYwrDHjAcC@Hi|X({zaY?z$=2KR&7v@oeZ zQyNx3of=s=#k)rFF6FLRIn}7GoI;w%unLWWkosEtkj%j0y4f?~p4rB~(k=i@j5#kE z&MTJMNlp2sb~0|+O%y3-vfpsmCqU;`!x_?zhS?ylig`wqqu!yLQS)76Fp*0?_#74}yN05fS{S(!l2ud!`sVct~R$#HQKrZV>tw02nx6b@h*HJ6b_V{|{R$$rA zGS+5nYbZ=6rw&iTQa|~UY4_{BHV<(oVw{_Z zPY8AF+~AE1Br`!3{w!TuY<^cbUbTymM~nkC=`4}QeG_suy+RF$4HXOwGx6$!)VUYm!oZfg0XA6hs$ers4!Huc!i-8 zg^_+kOmhLKkrR9{FCFkf0xNIwSsnI4lY9}G;&6qgX!Z6-VO)t7#GBo`g`pE|te!Uw zQr@H*0TLes^>9czlJVm(yRAA7l^+zKH1JFQ^fOMIAa1#PYau(d!O2YZNT zYuM2eP}s_=IU4O8MjK1Q-N{$X&lGPWc#8Ly{3*>H-X{6#*87>U#7o&4w1x@hQ~5QV z_iGiPB~KdDs;4HXwMtC+{{)>-0b+(jSom);?toTkWq-zV|uQi`$-$}*-x)y0Y)^a!uWCECrhywj5Mc}m1P*SBqJ5w;g;>~3547a1xnsJavY+Q@*&~PHfC<*8+>#Y6zbLXua+vPrZH*1JEJ(R54Y{S2&79`6A6a70bkLQ1$4~PBP zAo;i!&QZlNs?gUT!y&JHlYaqP zCnx!2ayFQK*KdFE(RcFSi}U*Q@@P1F2P@3e$(^$=znjJz`xb`lSg6ba58KrJeZ#)0 ze&xQM`to&~(j*7s{Gr`HxXC$tjE@6E z8naD-k_`*3&XxP)5Vr74?okLLUz;rD;K+J|e_}~H*6Yd$tZ=Ri=2RqZd^x;kJ`&%mC zzrOsQvA})6UAF8pqp}y{M&x$1p^Zda$i}WS8+Q`)iD#pIfC}_sc<-DI?;cM65Xpie zbPC1w*&4tAD;np{x<2`R5N6<_Tk&jT&n>bGNtaH;P5g;xpZ4s+J~nCVhd@j(d-%-* z@y(etdrYwWAT8(-j*sF@_KrOW;zIJU=?%>jCLDO;7o`ivz_%&N>UjdLI@8>flwDw!Uy7hei?%}(r_{h9fP=TF_W*xoby zkF3Ym9^bRqO3KWCGu|^hefNTX&OgoTjrNqCd#I&z_kvz@_DP2DfZrSSoA30TE~q%a zuHCn=ut1eZPAuNF=kEOnx9q$IA12AxrRz7PI4GfUM$1ON>&?ShYKh6GB}0h;``Dq_ zi?KtoHw*=^=0g#O_@U@$Ih43El(;gKxEM-2b|}OVaB#FLqnrH!_8J0UHeC40!|=@= z9K^KEbWxaLD{dgA)O4M8-AOr{0NtJMKocq(8DJ3lbtjBWz@=Tw&*6z*BD5t9pYEZN zo7uNt7lg*d$Pin}W~Y+Ky)Vn4DxsGy+ngwxf@?OHZAm9w1;8DrP}S%S-IuIVLvj^e zYt$9G{2+|e;!`RS%FK^r)}47*-x+Fgm38L;MEDw!CWFep?cWcZZ_?%u>;m#?b*PzJ zC=;HrHEGDx#{+vkkv+8QHf}@aUl%_w9eq`3s=u3k;ZK9?3xDB&wn^52HfxvUj9B;K zJv~Cvk3@|Ic5W z#NE{i4yxR})8<0>q)~j5a#zfA^wZ8B!wd1SriFLpOJlv0%f(Kl=CQ=v*fD=jI}%wi zR^T9ORf5THO5A%gib71gTpQ=^A)_B>43}&> zkm9QSnjC#$>fk3eY^xlYyz5Oq=MCXqhU z;>2WBuh6813LssMRGwrX@-V(5Yu~2oZf2Vm(uSAohTK7#E%sJnNXqYm2+Fjp-k&kY z!0Bf)sf&fmWK0%raSnc66&mWx&{5VgiK+C^!e%Ha1*S2C<2#03IyamUrQW;=6%yXz z)`WCy;*b{{;Ul*hfAa4aWYyvgcLwaTr0|>h#sn(a|6fTG7K$27%9v?P#s+j^OSXgd zlnYZ$Y6ZtS#SjCm&vYNVkW7wYs!P)3FngX%a<~OvOxp7K;FsiFCd~7ELIg0O@+CX% z8D3JkGv=y3*gLpH(n|>lWRIPE{FC$Qh$s|?VgttRC3%uJfQff(6S-gdrU!>F*o6t4 z?qeU5^}F`66$8T7#di!}C}}$d zI`32+Jk&j}v)ylEgqUVXVfT{}pm5VzWDM6(S?FjF7gOPZkXh6gYeqVm{DwP0zc>TN zXp61a8C&AyBP&oO5fw+UrX)!FU zQD{8^hR`8j>6pa;BqN90`)2br{csOJbD8~Q`UeY~GQc23A8RQfgCDx;$xH4D0HDD1 zr8RA{U#n}Hf)wSPD={4knuGZqaHS3n2BKZPaf;IzV!asRl63vy!3!C2BUTj?DGLDR zt_17z7LcMN9$AA))I%hjTR;Om3Wx*5f~?o)WSvk8+Srsw`a+0#!>&|qvA5AB5cB$! zd-ngyiJ2o(dIM7PDYcZS*XKkHLAklAbV_?;Pl>RnG@!JxeaNmMBh0Yg2-_-M?&7;Qrk^|rlmNhEc8Y=k;YHw_}T}9xyr(BS!`KH!1Au!|#HaHvg{6#7uJJ@Jz@%U7W1 z&Np*@~@Rf70KO-|@dU+YI? z;MMq2T6L_hLs_M)1W^1ErJ^WF5%~TE@)w>9#nOM46w8haTd}~b71d$}-T%*Zi{_V! zFc%6&_i}(dtKkKCqCXdX(Kmz< zT(6kEF};F?;Pj2@AyYJ_hfqM%fhLrU#gU-!uWCfd|}fQlF<5*jReZDeA)!9Wcd)+)1`&^ zLojt0kqv(GWbd$Xcsu4yWA?>F!Gb-deaXJUYDBy9qG^0`CEdY*m9c}D)ugc2p3-!& z;B-&bL?1RVg2Xr&#TciB8N>?tVP76TShcf=u>RA=9zMq-#MHpIB2-kXiZ@Jpqe(Gv zvGXyKJGLS3V^7Gj{WVQL1k&|5kdLxwDe-?vUp#2zNxVQ8KA4Uk3fI2{TRz~w`CtI3 zG{^u`!|Z*0)%U5!?2_!L)qGS_vO(`6PS${v(`&`a;%aenvWk;KFNtI06FJs9?bkEXW4CyYM3UX)M@+1uP;4asK^!w|OvO>4CBekawni@pn#NAHIb)o?-1heCWN-P8i}UiU zNpCrM*GH?NC#}@$WT(C$CVQFt;)LDo|B^WQcRVK(`8N)3cn%J62jt(fKZnx)-mreJ zdc%LL10@Z)18kQ{47mxDk{FX3?qI?Vi?c(RKg}Ms_PpfCo~+(ERHqub;D5D8_7y83 z``Yt_?CA=!g*UpeR&*nAS4KA3*afnk#2pVmMZ%WC@3|$5d(W9HY=r~pBk?iF9$t!H z*h1LdV4|T|;VHGc0#9!|PiXFZE6~h<$051|0Lhx)dW=7Ct{6{EBzs*h`2}`U6XLLt z-ObqD4Ds6Rc%2~hZ6WcU$jjOPP@y~y70ToOvnY>4PI*v(dM_%EXI**R`^B>;k5Y9U z3fGe+_RMt0aceHu9Zy&9F6s_;kilBN5Ol`_=Smed)$!<3^j=gQZl@q@OX%Hn=LMlU z9zI8euSs(}S*Hg{_(I%gx_RQ0*tRx9tE0TPM%QBI?vstXHM*LgFWee!;qr0!;}e8~16f;8KaTN2xW-=}~UCA?fwc~&O;#@z@t zO$?BHw%H8h-XnP=!G+W*pU(hRq6PTIWNw|0Xn{=$XwKe63#32Iem#eXfuu(@UO>3O z%}wfD2Z9dOZ~<-+vvioWOC131_%a$YVDgVC0Zk*#4I(C)2AxPjjlL#U1N zOo}Nm#S(8PsJqOEKdJ4qSXgZ{@`AbYH{|Y|p%s}{mItDcu=^F-ga(t3`E2@~b9V9q z6N5S)!e%Z4PL{qH4;qKuvGJh&CuIn%As%!rTo2=TbD$;%DzTua%~49 z<>g}J!R{)47B%&JIltPJNYya`InXW^Kh~->PSzMQP8iU8x&J%p=@p$T);?brF4lKoEutsHVjJl?2f)%hlcADVX#xesw>omc)9^vyV zE96M zTpdeYHGOnrj^xp8Xyh_0gIac`O%h7HMkecdj7E+v*#eEsX4MUybj{Rpwm4$qxU39K zE!DF!;6>E zYF8iY(tcs;gzd*69h&48nP$P8N zEPki^)X50+hAp*}olky{<>wYryn< z+CH3}3APXI#4#QyqBh~OhwYJJoz*~O#w+bUwR<{f5i)(xFjWSE(m>ZSP*of#nup2d z_9&?AfZKy`fO*fG8CoJR0e(cJU+r4@NU@)FNjlyHi~N!_5W5)aObI~R$oBmm#-weK zgk=d)glSX_Adx}RCvi#%B{~*e1R}9uUSCO)l}r&M$ zCSNW{vJV-`dLWTXWY`@gL$e=P$zM?8j2kgHCFR_xaSbK}000v{uf*W%7H>@DrrN`ux{M}J^g=TEuC3e}p8mOcHE&k~9Lx{m&)l$_3k z3jMi9MpP!HBKmK1#`u(4ul%q`G{zRtkfl(YVDKBE`pp?P7@_J+0O8$kEIVb2Vnb~# zH2_$y{1kP%N35ZBl_Lz3uM3H_^4VwpD+I?L15f@v*Tl=c<%l|(QPW4AcPQFSo>q+{ zPP#XbRNj1}c;nu&ilFLpGBmg@5NW8+4L%75;u=fG5V?{L*O2k#$a6e#4%bP&O)*Rk zjiCeU)=rlZj^YSiqM1wewuC4>kyXSaiZ8^`)h<25W+^$cqkk)uY{^dRnxlk1TJ`2A zgKKERhYU}I>)%?aHeN^U*kfQ7^%z(UQ#n4;z-|q3K9-;jyugL>78a~c?=fcgSWjM> zJyh#SDF6|FPaECjbJc7uBYe^-7n`urVLCJ!)z(t-hGjVZjm>1_iywbsL~OF3VZ7Ot zc1|hGp;YO}g)V4L?N;e%e3s4H>1ajsd5M{{X5(3_qY66mE>hJ@S}h$ZCCiyfq@tOt zo1K1Abo8R^fLJ`IBLT{R>L@`Vp}=-J3neBhnF~@emvI(6myKv;mfZ;JICRd&lFSkj zQY2q?CHb&U^1=NRgr1T1a@aNLuZ&Beu{!NFBATSr@hL2)Ss9YMfe4iVC_%5Rz6} zIpK8|>6y@zk&&yZ&|@d7#0~;EOLLtV1Mz7wJ;xe{OzRD@M`~Z66vHIB@H=hqykxuQ zqFk!GZ|6ZS9slxJx%9~Sl}pFhD3{V)E~U%Mr82;_I=Qr^luOf2EL~fdO62~YT`Emq zluFN7Ds9Q7(iSH^t~T*;wYy2}COr>Q$=6g!r9)q-kovMx=}9p}_R{P??dwO4O>(Jp z#NK(SbYS&Sqo|~2wD;u0g3d?}BS^Rv2Y`odma_0U+1tgyQ3(87^a*3UMU8Spr!7ICk(KkPnEQQr33$|?f#%Gw3Tfiad!=&P8j%Jum(AjZ*X5U8_RReKy`I$$) zS~t1a0=`)0-tzjqmFsiDK3bj8tmtzN8#H|$czxa+QRbYGNF20SCK{eddRoyESq zu3rJg8_yGpJKqWvqrJu?ri_E?9P2=MC6uLD-M$I6(!oR+hdR)_$<{(P*M zMW^3%VMl-$$6{rL@YGY|_Hr6$5+AAJw#6vuPRn9a>^P4x%2-p zPJXSxDJP@#OLw9h$0?_(IQi(hrh>WJ#=qDOeVO&g7)c12+V(ZvYv~xl9$PxLg455^ zu~nw3w{Zr$)6g`9ode)lBr%KJm0{X z_Ef=1HhYF`Kjk>d;VVmB!Dw8f`l%|?m=>)pB5SQqh)mlm(seB&%Mt6>1ZpDLmLITqz&rTDDO=<( zhtr!#@$+t7-@A2e8Er298j#vG7vH0mkbMsqe_i3?>ztU?;Z=*6TxMOb*yJtkR-F2- zcj~*jQ_rG>^SXE40V9Ci+;0$a(2TV|z9J1t-sDtVTsP6_YNF(HCee zmy#@x#^r7t97W^k%*%L9N8-r09F2#bAsWp;UtEgD zz)9jUv3OET^DSA5$u*9n5u0u-*wm*m6-$M*Y8QshmkON)$(?Tnk{R$gG}rv|Z#}+` zoEyGZ?V&3xmwZU+iuz)*Ke(vGSuA$Gl=wYgt5{y=ti%_pwV=em@BN+;OGiy=>y_MgO>`fHz%P3c(Kg4~I>zYzL#MWz zVFZY#m(5j(O800qC;s*4DJTA>3p*$NRLv?iXM*!Bmd%O3Kdvm}tcMp))N5MCFj)cO z&&DbQA3Qqm>kxGG^lH{2=q$epm}W@$DPM;G*}A!>Wug3JI1b8YHXK_D<-pN+xqx~F z7;ijJRNbji_45

<40+)c$}Hq=cUnhpdauKo$+$eqzskm0BVc%eJkIkEi#(aZ&6L zGFtUgoFk{@3l(0>*%;zTeJM^|@Eoewby4uR?a|Lg@GJ*s6Fmo(!ug`;IS)`?R`|>t zz|Ssx(y+IG={mq5&+jxjg?6y@7JXK37a`$Mo9Efk4j14$z+-hnKAZ4)bXiSNS#!2n z#nmS#%?54RRv4q!=WAEOz_Vd>x|H4XhlgRWdb-*YONam|Tbhhj1!JwD4%0HsX0d8? zJ{+_oCE061X4e1TIf!l8*QR4?V`ReB&DH8t_9%$hGrhCSht)mPjT3G${e6q}Oy@}W ze%jVnJsd1d0%3dczv-gVaqbZ{y&Dku~-4!b#_X09- zWwWJAm)h1*8&-^5e`eQnjjSz(tSu`d>uN*R)e$p=1-cB2veayoGJ9p4( z^0ZyC0JV{ZtvJlzRX3$ci|Sa`WIKK>N8#akdmnO1^QqRlTz<~wvAS`=%YwFKK?q7p zS-=_Sy=7QDmjy!q+GIgu%VTWoX}EhQ7iZrOo0-_f>IgnEw zPgM0lcB-mr7*5;G(mxX_RJ(+bpu+ios@Tv}?o$AsN#zo7NH!`zP_4W{qrE8T-Oj3A zAB(K+W|xMCvy<)H*dAta;k$l(F8LXJs=)2;#O7mKtyn+AufAGQ+x288zp1hO#ZOU? z=w-RTRIdvTY;{%d17X<6TW3?K$i;hTlt5D7U6M@R+?@lWn);WuGO`?aMMzEu(I@8X zal&8}G8paVvo+eloAQ9ch)|tYQf~AsFCuMy_wdt|z5!=8TI&y+liKu5zmoyGAQF^w z+X=FH8<9iZ1=O7HZF4T2C$;_p={&%K0;1mTap-4JNtFDxw&MXb+Pf!H%IuwN6v-Rz zfpNcBsuC-*>sG5AG6Ks}TKKq#mZFJ86Fy#9p@m-w%ML&R4Nn1D%*xWG_b3)k9~weW zs7nHh$*@hMO^Kbzr9Bqi2OUS9At(y{#$zmN%dy2yL>v|`+NiIxtHulu} zS5jzyF8P9okO-!P5L$uWZjCN0AQUMm>>zYx1VYi}QQw7_8Lx<%aeJUuDD8Knu-J04 z1QJFWdvJ(VIEvSCj9n7$8gqk|XB^Bg{S(}HK{M<)r$6w{ez1xq=6MH}UKM0Wq@YZN!0WL#=1J~jR zxG3CDQS**3$;T33QW$}v#y04vqY~%6{}6^nz9Yh|?T9Pu4VYvwn1miGkm@E@Q4xD) zcdJdwJ3L&vQQ-a;B3jrZMeVgKqNP2uNV^Sqv&X97$vQg3rwy<@VRw4>R49s0d5X$A zlpL_O!>K4zqa3uPM(gsdE^8w-db--EIH0;R+`e!CJyrv#1gPs*@3+Dz+8yPyQdL>3 z853bFN`BUXMTQo{*Kxikzii$=ocWKaJS+m2;ry@A=!~@%#`f z#l1gZF5I-y;hT97o);|L`@T|i0LZR~upO0T#GbMa543fXzqOsk zx{xNPRtG6Rpdir9=AjW7EQgyD%i-oMy*f9FIl9VTom#F}|8ouc zlOFoon%^;14t+AyCQ_WT(Vs2up1ZUSQ$-Yc)^!niX;%tvNYmbxx?x zE_L$5*J-k0qO9{Bby3iCl_CmSIp;xHB27dn3T9a;z&5?=ZGCBHoguaL=MpJ^>?@aE zr7c-rpRIHWhVwgOO1p`*?2@?7>92+^!7bgH{JarjG(~{g1e8o4Gs|?~Tgj5~i0PwE zA+vq8vy}ro7;Ymvh>b!q%cyw%PME`mW6||EGq%@*+x!vDFAg#T5LmenE5&5zUBFXZn;q@3_EuV7b_9JYi0!_*TJ;QW zoMRv}GD{oDJ00Y30JG?3X);?B?*%9u$d|$h3ThcceO??0dqHj$Z^w+=nztk4+1v65YDk6{tKN<*ZSU=r z5^2QSDxHZ|a+mjXuy(u+Hamq`zO}k{>SP;ms~9u-sFg=_Xd*V z6Zj=(WeD~hMEyhaA1#AIj^Fyjww;qeaAeR@07fEz*p$0i2z`7!34LULP3SL&%GSAn z^3-z%O5?WY2`Jm=0?PTXSM?uq?ChfH9I0|t!CVMbd5%LFwWThEq5NnqfD$_UIhsk{ z<*X>#pIBF7&ah%5ha)z!&8@?#{AFaq5DPX}mw#GtlwL)tm&wc08&fVuy=%?1+xwl!7 zHK6mr+S@pDF4#G}b}IOa75PNPA70Lxk7nFINrLfiTBIQKY9|J2ZhR#uQOMuI4QcBT67Xjx|3DeOfyoUypRa`oJg}E#TEU^d@b1l6_82D+0*aG~F ze-#s!$E%&I!d4TN7Q?$XkKx(f{h@Ac6EHf`A9}#^I~f9w3gEf3X;1HVa;LlrJ`9FR z5d^;I<9tFbDlu}XPGDuA3hy&0X6`J42$E@6m?8=j<}-Jp&D`;(Gt5tzDjSN6h75;n zPAxE+!AMrWziC?5dIa;nPZ7({+Pd$p5P^&EpzHD&!lBvB}R!X6dD zjan`VcB)#0!DxPwN6cM0$lD~kA;1e#_f0X|)ytBBroYk%BnlR#`Wa0k$mfuH%4@Z@ zft(GS(wiV)1*ppD!)U7OwS6E{ieR&Bk+V)HjV2M0)mqLSrS0C1nmOd93~&fs)>c@{ z+LA_*4dU3qiv|P9S@Q}`R_X31N}@nAgn4xi1!~+UF9aUhq2$jhA_KMLg)9^%%J2a5 zQcH3ZeA8)|=O%?!kt72?A=L}Ki&ec_j%fD%1f(pKvtg{;Fm5I#nJ+N94>A z1Q0&O8*|CenT%>p^}-+s9F0jAbrdAsz$fLDWoKj>xFd;jur~iswT%IXbS58`Uo&h; zq%(!dK46@!s?|4#xX^$Onr-^C?H+?yt~L(0N~UzAohgos*jb@;9Xu zPONcAxROz+7W)4sqyNq*{jISo#M(BlK^%Nu0|q`tL7K z|MS(29SizDRoznd@8YnTYjX_xTA~zMQYVm0e>$6#4zjczo^Bv*xRj55ipGx?H9Myp>Db@#yO^n&e(XUqk526{N%$XOwTnmg!c|*UdIt z0`Ig>EbC^|Bp27vM=PmLN%!dE1XN2Fy%p>=guPq2Xfu4KwkpWuPWQJO?rsIhUUc=< z>y)FoK^?y~3?T=i3iQ8%I$n60Hd&Wxz{~`wgdNEurFl+mfEksf+5ma=9kPgVOWxUW znBBq@Q`&Y--aTbU+51(N1-_rC9&rEfYI-`z8=m(q$<*iyr=TF9XeK*~?fSb(u*X8vg|o1eU(WjmoOtAe0{F01 zsFwhw?iv6a1&f%0gV`Z5aMonsebxEMPp&W^ECdo(WhhjfHye*kUbFpdJm_o2lkN87 zL9gWFu{c?hh=9IU?Kga7*se7nw!Nu~YcW+39E%ddsyFByZ1Ig8`ls+dfup zXR(5%XTifiS#1lr!|vz7!#`cEbtH4ryFU^u<3h3~aPokg%bW~rnUkxHldUCl(xN&( ztf|n>5Ry|2zj`w^3x5bF{fhA~vHxZm#ew~= zp)o0vw1H1bSFJf>sy1b+9fS0}t!w0~L=;6S>mX4xRd>LST5+tM#sGX{&Fkcfng60N%um@dL@RF2ysFAaKcuk%EhoBPp>@@Y;epGL_v z3u79UWaHi#6i82nIoU?5q9!D84uRhcQlD=k1Hal)j9O?_dD^`Y?N>b!#5(d;2a zX^U;@P&@(*>&BuPFeE@aCu1ekm?R$1(Q(^F!7E z@-E3AARy@in`ulSS+gQh@^*#gFc`99)^mA9EIT+BK0QGU%vS)mkA>v5PobbSTbM=H z@qjBam_T}jXiV~V3sl7Iw4PmCe|8iPIj}9?iZAIGB!8Q8Q9@dccL?uyZXd>T8Oyhw zg)<%fSQ>9rpZrXeE4Dv8Yq@>QURDdLwFk~#?EaBr(ToR-c*8@5p+L|8Ku_izYE!MT zcC7FX_)Lh+=RH}E!FG3s>L~e)W)^#|hLnx>2Z8V-pEQ!amTY$ubo!n0JEdsyITb;# z>GyxB{61+_)mK?p5btCQsH3+oEBTl`{BKV_oK=|^jUG4I)O1+yy(cPTdu7gnmk5w-I>bIBKrcc5 z8f02YRY%m3nfk6MhXT@qX1O z3U^_hy4ISkE}1`vMf-)4O3g_HbcNb1}S~Juqy#IqTnH zYu|i1SK!Q<7}9VhP;6<-6>MiGih@>~rlNSKPujj^_hpz}a!2+?YXl--d^9IXh>eew zVa}`ff7Oi??>o?45Q-K_BSDCN1)>2H%1ELr(uF#Kd$@x;e!ONAf>_B_9s4B;{Uw1* zkmOq@Nfmn3ZNIEjSPJPEp|pUD`QmYb&E73elv#tk7iq>GBaEbqkywaz>mt>+n6!a3 zGF?oy<*0X;Ib;}rq1Qpg`RYS+12aM2umO>KQVa;P^#kHpM^s!6NKb6~(htbOTBWtt z`$)*5?r>KYvKc_b%~yk+=A0J4f$weGA57y1cMUtF#T-TeWSAa~W;=%w8)&!rJNXKl zEZ#&j`sd82ND|;SDvXndB_9TDS2z|eaTE2t1c7dx1kvV9tn~m!3U>1~zqGQ0>RUaQ z%JZ5BWDhv}XthUgPsI_cSkT58#x@NzhyiJsyj9wf?kU0^i?%QtWuz&?q12&!_0>zz zIP4?@wg3jXCpz9;A0~H?v$5&Njo5DVGqmvokekc~kZH|iTrpN|Le=agW+dy42L%a6 zy@%UjJD-eUJf0JjMxyq2CJ8Wn1cGj?mgD%5tWm_Z#+S${b z$Qy1qB*Q3!E@RiS_J&rV&GDn3|JacKxGev%Isb7*{^RQW$II;pmLgs2W{UyadC}xf zIwNS&zD~(z=&L9wkDR@Pd-U*_A5I^ zK=CvLT=jVlSLUZ9t06C@NpHB_O`#sYjm7vZi=e1SQrSW18|aUylp`I&;ZlO{^ZyGg2ZINcD=@RZ%z{ZfoL} z&a2n$TCXRl2vE9Sqj%`_RJGR}R#x+~bXtjr_HW`evhpNxi zn0h@$uX|)Yc$!sznIU$`sv>@yX4ae`FQ(G=$cvb2#E4A3a|w*sl6T3tp-E{ho2zdy z`i-0x#)uUx=_^Yh;lf>?i>;LJr<01k+MW0kLt4g|VPVihqM`T5XUsXFTso`-KAJ}P z>IiX!z~w}W*IDZ9W!$cNMYNF%ZIF+xHWGK&&z>}w@ntr&?6}?BZxG6J4VSCS1`)eK z7&VWr>tL-eSR%)hWIWh-R4o>C6&Ef<@6p!tfN7l!zCc;_=)Njw9m_sceRe^z?ETdq zdX_y{eYQ51y`VS~*Ks*+S{-NZGeG2=xs;nuS7jS4ojc^cyPGuwHgaB#S10mFWKGxg zpr+K#G;z7&e1ut|P@>OVmnWcy7&CgzzF>D^lV`D+ysfOg-TN+acCGn|5?|)3Ak0o< z`^qMkR?}0AAZ$G82HC!nvi^s+ePcFb^H03(8*^}nf8uT5IPT8bXVbQpbv!#e0t31)0i=uo}Obtsl~Bu5$}lo+-% zjY&k4_$D9Ic}#5jW{pZsn1c3$9+G1$0C&uFHRlJR?@cTV#aK?7+P3gZijHHy8mF!- zBGZ>KKr6C_U1d^3<{f#HPhlHAaweZLL{Y%>PWj-Ri-2;FiA8Di5Uj_2xppiAFAXl$ zS7@4oI&clCq$CY_WqG*8W(`}p)K8tajw2WpVQ9P`VE6%fqv*A>eeffE@KSlXWK|T= zczKNwjk6a|i%FQ9*{c*R;#Wk+?aABxy?K=iiI-Qn?{b{I=hS_NCTjBO*yB$>8ku}@zxdL9Wb&!^ z*@#W?qg{m<)uG~n4|Do?$7|KD7iu(01zzS2*Ztf|EaH4O@c<)TZm)p=$~sDj=o3Nr^%cs?yQA9BE~3QjB?3 zcRCv{m^EgZ$u;o-O|a-o;$`+=U-2My59W&p8{LD`8iEx`+=Ekk(4z|M4F|;D`@;%u zrKQ3gx)xm#~gPbBEQ3f->i+?NlTzAkEyZWyvu%bIhQ9@2X{N4CIvC}cj61`K+OVHv>R zQmlly|U<(IZxPI zDd||kB!O#{s;p^c!`m-HYN`80DJ@s!Z9@J7a?3Vl(xlO+Mc&-zsSLeejl=G4`W9+B zV=*5;6l56Db5@W84CVIWA2I|$Ue0DwG4nSnD|kwNYAh2`BgWv z!MXt=&jP?WRs(_R)AxWSV4b5vvCIVJA+K2PnihVb`XMO`ul5C+*)F<6-NW`Gn+VSk zxgkXJZQCsvW>q$`H@t!w&Zo%@7Z2~Vp6e1@#GgIMG%w=+AY^g-@AwL49o-A?QC z^N$2K$lMOI6dE*R)?5itm^Ng{YTB`-$P$-r_$XWi2YzUxtqLY;;WQeZPQYywon2R@3r5^fuKWb=EpTdI5`pII9X)hc6~53M1nMICe0 zgTNH^aJi9VC1?Tq9HAcfA!Mo#x%iF&j7rB;tn8T8L;G~>u`0Y$T2gQe=xdJX-4ToJ zD!>Cf5MG$OmGZ}|)3vcM2%`UQ?^QZjLPxRYN!sldSwH`nFb_! zAYId@5~I@+wxnGYu+ChWPFAGql<1C?k5{Ucc0bR)HDqN0&YehEfR zm~Uht%IxqutbpR_8WhK}#KTgfp4!4IgR6}@m&!1~>58H0wdo=G4Iy1Bq>Je$I&$Nr zrl=WvDP1DT$FWMxf@DsKA@)qAnJAzQ#GWX}OA*bVTXaD}uu*5FJ<6~tXaFllw!xiJ z0H``UM96kqr!ww0D$k!VNSW2aPKJ`@TDT#TK>w02o0{fxcDA)r%yUwn*91>YrEAq0 z;jFprrJ7U&FvP|pz(I>-OCTFuUV>E$L)R+Z04J&|q?f3O7>H<;?M~o=JQ8u53)tu| z4w=v=7c^0NoYYlzRrXL-ZT(@ZJb~aCM!ui6rpuSjr9tw(d_WsekJh#$T7ITVFJj`_ z5jzt_5^h`y;nuFG38-Chv1ZzK3bs8bU4GRK?4)cM5FzuPXr*f5QqN5Jrq4qS~i1 z1NpOM*xr^o-Q?<``htw~UYQeIuONAX*~Q?h;G~dueZc6lh%nui!QbeCo1i6X3G%lX zSR%J?N_a&O%G?5YR4gqQVV^lIr{DVTNu!5PID@5N|HrCS!uX8J8LuDXr!+>1-}v%a zixiMi9wH?~jb1cdes~!R$)ig2KxvAag_@2wo6zGC%P@_jif=4hXWyg}mWpOl6Fe?K zW{k%XP8Eo`1_gK1J>0Sbhw|3^oSj;4i$C~+hA(rwur#SEXA=f`Eceln8=8aH(9F%q zZ8|*_q1QFaSK<{J>9+;1XvY1k0t84P8rAcm)G$5sn$c+3hEQ7TM#iQb`OX>Vl2HJ| z)xaz20$0w?Rb_gS2&|eJ91$oR-TvEV44`W9I3%7rDD3Q&A}MZDSqYK4&7d2CJ7b54 zHG`-Roi=&(Bodi3x&T&l1X!jw5ln#F30SNo1j`ZQ5mG^NWBTbDy-~I2bVZGgVd zY^bdTG~|`})lqCncGu_C61O?SHjH7Bw7><(Q$K^AOFnL&aj1l^zMyO;z%QFie%Du2 z5J>O64V*9_BQfr*XBGiFr9dd50|8-0%CS(w1R5b$Vu1d|l#y0vG)_ae5Dvr6qIpdA zDsrzFNaqBXf(CP}IpzDR+^ifQhWX zqf&$v5cL3m$`y%KgpV%S0kTd7_*hOryKY-eKQTA8DT+wm3dj8~kfgaXNTigt=nG`G zaiC+fBPlPPwEInnO%gV|7ugoWqMarAx!f?>?z2O1)es;Z4R4c7GJNszb=Ync)jC>k zin}x>sE<0#WBfw}r$nw$$NLSbriaawz}l5mQ;5jA?EPg3JIo%H-&aTh+(Uc%aP29A zx%6LBxDGn#{dwEFF?2r4{yXXx32@{5P(b~N@6B${ytdu@G=vPrt&i)LXOK|hZMZjFO;)7|Iz-0_SO0yT7;^EC_@q-el0 zBm96nMzr+-8xV>CJLuSe*=S|Sfb)+X)py1WkEvJ#gTqX9lyGl-l%g9C6cl6A{|xDF zGrNUsIgxiW`|O!B!FIwMa~aYe9%=b^gdQsITHos;HBsf+ODs^~KUcLtVQ%VQM?rv% zU;Hm9{ul9>7sdV9A7_27X}S};*ctyfa9$a4Hov%&^(oi8n*$5nyaUd)H}l+l^GyET z{GGMC9kshR*6wbr-TheY?hUoOTWfbeT)TUH?e2B8I}lLm*=wtJCvN2Kvqi68mA}hc zP^rj#CVwJL&`!f@AhGv;#o_7-0D&LP2tRW4B{y=}|2&oYX=iGRd}H@Ns12%;#9o~w zmx)(#>6{dXRiqpIZbTU{C9r}Pu0Ym=7vIR9Z)Qs*HcG}S@Q{~5LO@?a#bomFnoN#) z*07}fnsn~{4oXa^zj5{ghccnfp^kXvR3nUn7=@wKzU<`5Pwm0(aa+P&#e=VXkpNue zto$&$Yd@+Xi<4iK6>W}uoP7XiQEPX@Y@zy=@HlU$CGc zq+`bjvx@8**N0e5X?%Sc^!`aMn^z!^7uv+hU3nSS`#=gLb_xX3)+IwrftX6bXE2LG zk{@{rt@6VZ2uEfh>TgHrGZjK9;ULm8iZ}%#BRtMbo?;D}Jk`f25VxjV3MBTu;W-LK z*rNs~VH*_*>poL_*7_V*6i9?Y4b`C= zIEtamZ16+R&4vor9lG%fJf%2PS8as?)gCTOF;0?ceb{EF6&Qy{1Nuv%hR*P;3Oy9oPW`#retu%qHId$G(Z zKEJ`0y?tPlcj5NlpIxqlHS--PkFeCUxtY@QK^o0L(FQ#WniRd0!Z2?P8ja5|w ziyidRZV9aJU_y=Oz~U!>dI1*!>IEJ|1`)Ox+m^LNlT?sZ10H~@L!v+YVLFtIds66` z9S!c6#?camE?w7yU-u8P_MlHSDwru0f>X`(;s@mg0Tn_Im!@6*YckVFa{ONW$(WzomsZ zs_F`Q&1HHp0Uk640nD62J%|Or2^>Y=9GZ*|;a}k+#BcR38dVy!ft;;&q*R+ekUqXn zjt<=`YDn9Ibwc?%S3?g8pqxL$)VfSQUpl4MJ{vj{9i8c90Yi=u2NruL&MjczY$_Cs zc!pjF>vXw-{nofl54$%DjNd(Z34QI|OvXHy8z(RAUt(rUXV^9d1_uzm8ud}EBJ$_a zDl7@5*;N)R_QSR*9=)RQw#*^{Dj!wiU{;>Up_OOZ)f5JcVul{?wsy||-oI4*`cknt zbExd^$i)$4DD&0Pg%EC!~&Y$2@bO1W%$l~1)< zfz?F(b6UTUU7Eme+wPY>)iR(7HW1cCO#cPaJts&bO$np&hh8f7;-2x5w(_-TbN-B& ztCqIZo)N~Z}rG52XP+~gl<;z) z7`7K198@VhY#CR!_`rC@?~D|EHbI_kQV*e(|bnf`W!SCcB8vUKiZT$PH3C*xZ?Ci%xb4 z;6MzEOtL1?pAE@}bGddaGXU^CLFp@lH)wiAnJ3^(ahpaz5#f#iO)we-TJ!Sg4FjwC zHbh#WPKmULtU)!i+WY$ybOVjdF8JS2P8M7rUPeoN)#pMgHFl#B3h#^>im3@k?I@QX zWsev<-zy=CS;D@PCD4VyD#y4?;ZU4Vg4dW)Zk4f$U`;eGXB8(QK%_K-BPd{}5i{Lm zlx3g;EWq(r>=hH5V}XY|R)tH)4OOtUgkl>iVDJN9Zp3`BMqV`@v{F}G!^DaG+I&s4 zSyIb|s9V`*sx+z5jiGh#Q7oZ#o8PR`x{>}ET0dW?P=pp6E}8*Nbd}jI8C1faiCRAg z-1;5~anFJ_DQX4aF+-8w46$;?C^w#B4V!Wku3O1R#2u1VOo)lOj2Jz0F(}Cw&s<16 zr^7drE&r^`HcyBG0;yA)LgSCJt*%C7Uj;Mq=U>u6u$+9<1R$$H73P&Gz3?*?F_ccA zz=01Yp7aqH7eRoXCJJ@}uUIhX?uq)mv6bs?L)n7^Lj;Suyl#?3SLi0a?+k2J0quE) zV+EObHl1*~iS!OrINvpQd7$&Sy^glEdd7p0fAAgG#J8B_rRk2mmu_|$f-}P$+>zaN zXnx-9CgTigx{Wg;^m*B6+9pRIO8`pr)n#KUp5E+aBMljYn{=e?IO#}eW0b+0m&dnU z6MRRCR5KB%O;4pbxxk4Mh|35;FA`QI1gYq|wEr6OFj9mhX|e}`+rL5)!3N}xi^QMkO(jx6BEj}5Vu0m7pIfxYdU+QOJ0 zFu07CCKZJH!t2A=GE%v5v~s(W%enfq|GLSy7kf(Lj^zCgp73wXk8@aq;0Lfex{;z{|IUMq9+NN9G ztSg*NWcP?yv#tw@S;h;d=hoEAEc#tc5 zZXwJboK5fM_rcIMa=I0J;0bHux9?k=7ZkA(+~zcs48(nyg|`n|(*YLRtywJCKufXK zrEU#=RAadhZ$#Hap$0|1aX1ynVFXk3M&R9+e97UBiq0Y@GjGKiQ8;3nfekd%{5H*1 zXYUTHSPXs{Ebe0Am=mRWN;_)&;*6uFvH$dOBqb2#W~_HF+Gf%Ik6|DPe0 zr)~5P=~=P!Ro(PyntuRQAVSYhWlquwb~y1j8su$P>}Iy;tS{;S_{3*BE;~Zwd6gsS z85fppl|E6+`9x;3Wi&?bCXXS2i!vT94*tiIu=BE3FE2xjFnI3k_`Z!yWGnky-E^3gp?w zp*e;4esb^;hNv1(;IDP*!}5B=|;iIiQ0Vrgas@AKla`~NV4m??|ZLbPft&G&rI)6F0qT< z*Ns_eAz%sEQUMIfM0hL`1PLnSA5@8{Vq&F=RbpllgbSfIPz+&*B4xueC_@sWKng{& zY$}9)P=aMDrW9J*#7qoDlq@Aw3ATPwGAn^nibczaMLVRJ&-Z)o`_Vl;y8uCmHX|-E z@AZ54-goc0=bqp9Tv@qFeAOwyV^kaFM;(9u26-JYzSsWz4Leh6W90^|kHT1~3@)R1%?bFVPu5rG$Kqhr(6n+@;Gqje$4+HdyW>?$+>xRk=+L6^E8){maA|s7VuvSInH*^_os;Aa@8DjTmrYMrMrJBrSl+d`nAUJC*YEcC7_4c>!-}wb}WIZaCUMZ=G84N0b2+sdPbG7*H2v79?3~L^%78FmcT;cu-8vp zd%c@vT}&26(+-y&5a@J!o^NdM{}ul$I?wu@d)=;~J#X8NpHGZ}n*e3&!hB1@_a=9I zoy+SjZEiR85xUA~h@L71o636oNEg|6AA7G?%p0MX+qd=Z;zBxr*YJWBvo1V{B_rD- zNj-?!)7{b8mKO}EOB(*TnN(Feu()c5hK6-O>`JB6~@PYN}LkD#kC3HdUE#&Nw|Z&iO2;tCrOSI==G&o!rJ9?^I94+L7UFX zz$j-@hDfI4?P{A7yl9$cqxklIkm7_fNH-ghQpl?uaP7iKry$stV)z&pt$=l&VPOql ztYF3F?3Kqn~-~ zUwR?bzy75C^;7@rQ$L1vmgGz-sb2&+I_h+yseGA)Vj?;w)#u%MiWd^)n^a$1dZaZq zQKZpN$svqDW+)<+_DVH2-(1Fw9+S^uDKuJ z)^yQ;{X_hE6S&DNG>bt*Ao7$9l}w9?=-mM_(U*#pr6=c}f83{uxiGDZYnP?zbr2J{ ziNDwjlwm7-EuWfTx03Y09bKbQvX18dHMmqoqEmH&Pp9<`NvW}WE3n)AY*AFl=5OpK z&+;o|81p$^q}LL`VCBjN%zIXPv_K-R#cmG~5SkW}8&;wS_33Q&g~xhi^eT!#m+4tcybmYWf6f{@}g{LfoOon+H1%9Sa+5LnLYv$za!&P3$f3Mbw8k$FEin0yt^et zBQ9^{7nY%5qrYU<2giQ7zRC4f^DV`l>wEf?KI?^`3u{jPw9eO|0Dj*0LsMr~A?Emr z%<(oB%%Ml&hY&(cvw17*sE}>JCGty!iMo#UEbqk4-Gf3em*K z02DjB>&4gCdPK7K3^n_`w~3!Ljk*Fc(mq(&uta-_`!OcAd#3`2ngVS0j_?dNdkcGG ztM?&OD&b{IxBLqrWYt4fB}(0XXZ0Xu_J-tN)}n=9Sz3*80aT#=+g*q=+c6;Te&<)> z)qRi6yKumcZvs=4DpTu?DF6p}n3%Ml&rwBIFlnAPPBjci{o1J}qNC5oM{0mVfaq+y zXqk#F^HK^h=?9WG(^Q&*Q=c^ z)}os%w)Zl1b^0oLFEogFicPYH!>hN-O{oa>zyQR}vVXUxTR^oZfp%-)1uybWk0^Oz zz9~1>Y|3KU|1mGAHq|5wN%c`Lq+d*X)$&dvO5$-u)ag|2PLY-}`URQ($bT&|eN(Dk zlj#6{jmTF6^C}*I*cl(>Yn!_loP84sHq_U(?mYB?A!wl*y+Ej(LddF6b@&7hxGvI4$>L8vAn)+AV0m5pZsKKjk&GQsKX?uoJ znG3QGP3dbj0$Hqlf~@Rj`@$!+p)zt>7}`_`3xlR%U-%@Mh?fdqEqu~JHG-0xY9j(M z9$q_qlIScuKBHph8#9W$qS)V`m&~viI_wbBgHhW>h0SY5Tt7owW-QA*Kgr>cSJnHYxYF z8H>BtGfKN_wr%i<{aur#R?odLEws8hev7nW8tv9*$5+Y9B#R;88MhI~HK6`=>QtxGFX-B136L6Tz zu+H#W+a7VVD12z7Y0#Mr+_MfH%ruavJguomi}v(t65kYyONt?8ZoBMBgyWuc#5{EV z05+VNj*Fec1cVz(Tp*(_f~#`auvuF1Z`e*Ls_mqO<00be6Chrb{2lj3;^y>3nS)zJGn!q$gWi@i&Px@)iaO#@x^V44mCiV-% z-dY<{P9mtyech(LC#Yt^Kj~ULmw6G}Hx_o88VRtMATh9E z2atD)z7|nns~jN_>Vkr)mA#FDTr!U|^+HPPzR(!T!V5kol2n^=04Gl`V0{vBU1feq zI_yvXuEH@i3WeTe<)LUg063!o@0b5)SL9NR}?cd!h# zAX)tDHwhR0K7e45HG6>i)j!fh@&Z)PEPYlG5Fj)ynn9}?EU6m8AT-}(25B7yKH7c6 zt<^h>3RQa(@|Wt&r$Tm83$Rwzu81dcS*Y++i_)-kniLKY%;Zo2&InVB)BnRd(FlQE z455n{;JwzRVI#~iY(gV)JD+#>Ktf}8Qj<#Ou@%*zA)a<0Xh1itX>>*m23eV%U&088zMKS6j;BJ_$-HY;)#&~+`s9j*ai;_XViC5ULWnf z$)i}xAg@-+qlWkICfOA+sQRGiRBb$A6!{~~1x&NSoX5ME1Bd}D4Sm!NCBXmQkPV18 zSYoPHQFNh%(Sj{5A{boQpa_p9*oT896!w#wJFZDA>R|z57D?1=8CBj*7|xm{DZ49& zNs__?CP@mJh8u68vOLP(3OuJ;c;1!ny%khx_f~MGf$*~4TcN^8QVWIm?X7T=$f!d} zEkq^itM2N^KAM9!qp#)d|GM{jDj!YkwIIh4O~Pjxt1JPyJ9P+8)#D-eQ{Cf3)%~6h z_kl5Q$R3Ix)jf^5Hm}N zb*oIcbeN)UDDmrau7z>n4__$NY9x*XriVdKm-!@qE%!x=3Js0{E`{d z!~?@R(Lx*0AY5!-mg^F`m|v|4lnE`iHRmc*N6$kVe{XqP2~TCMCH58rxotLZ0qTHy zmOhFbdq+)uBu9x#S>w#=SoWz7v=d!IT5j2rc9 zL9{`Kt{8Xq@veaKtDUsqWut{9Ik)TwDltUJ1Zx0LeF9r%4N4P}6#I~fNhkK9#wLZ> zCSsCeA52Uf`_N*O0~n~%I(}2^!vx?(?86c^X>Qb;8ma6$ju410QbjQ)A*MftKt$DQ zF%m-IaP>|v>&Oh;4s-LYM3y!%v29)7&)fg?+FRf0kl?x*CXjt|v=h38SRZPF1fZ^J z(S)!_73%B5YaKb}jf7vVF;{>_`r?vf%DlpVuw;vnNH)3$V91$>LtL3XSHOcPoK|TKJzZMZ5IL2Vhgitkem3aX&(=w1LoJw-S(*1MQzgjBI-?kbY@&@zA}1o9zbwK;_$rL9qC`%W zqp(##6%A*v26XL0<{z*y5|ZbG3}4 zC4%B=EJmAV^$~JptFpt5EnI;v^dl$>`%*WGkQEO^9dClJQD+pW6WCy})@XuQi@m8q zoe)lH)PYder~{ca>Zov$P=}_;`;@RGsob6id_; zJ(Kp?_mY3z?aF#l9S$7edP~lxIp`~A&$$tgrr z7sDdObtj_HFH8wrZow#--*r@cxW z=Cs$&H?{jraS&`EPeBk`vGAu$5Ol?GxL#C9r@gvHPc@?Aq-sxl^*c!lyi<_~wYcn1 z3rWe{(_Z-vURi&2Qc?L%lsXc%DHGoZenwg0qcoB653Z3;Cs`sSSPeq3GFX8OAg*r- zJ+<|<#K2@R3E8!wOGyfMnm$X_kWNg&x|OP$;vnS!V7EiJnLW3yKzR=7OQ)oR+qoeYfiFd{(~p71_(H zPbmVR`i%b4stLoL2%{n`@h_&-t~=TH^;7GT3*I#q5oH$5_H^T{LflYrA0)~&$Oi1x zR6&@EbR%u+>n)}6LZX5c_s(T0Ihgoluli{TQ5=8vqcZc-u!<}u`C>Y>%&;O4B-s9( zBp{Gc{9|xFd(9kniYkfV!mN^f_*Gz>N!{DHjBY3NDLD>5Rfl}5qOn+`<&Iqv9-uU* zdb9{d$$yV&u)~xDh_F~SumVY#F(rv#VU9SI@5D3ZH4y6(87 zG%mmIj-K6-8a;)ONsej@0i~b9!fv-GMClwunjHFW#Li5jr$_uoR0&?ZS)CDA5U4=A zw{ig!1u=Vvo1O>6y6)j%42IiPK$Y;Q3o-KG9|$fJ%;ux&;i&-EiSXV=H}0Azy}424 zSw@Y3NH)J?m(iUkQ)I zAFL5>f%w;#7|4_$6^dy?JLspwE0W@!!Ye}Yw)TYZd=15$6)o+_u>jVnsB)m zQPL{Sbz#6{Ey6%LSrU1f;pziqA|Wrlzi-~%nj-u6vi5&+ zp!CrscLR7PsZP};;seI9j7wqdK0_w6Kndb=q{EfSo?7lkxZE~tcR)1ddADYH)!)Qd zGMq?0XqJIAhw&et>J3|Gdnc#}yiY6yg0q}x4A!rC2c6{a8MX$e^+w<6Bxm-#*XoXh zX7JuYNY-?y(rP-xn0J}K!{F2=8mkNM*Kulv1wmX$ky7#uJE{(bo^sf#I@UUBYjdbM zHHz0)v@n^hH~Q`W^7htuK#x+&s(|50mN~*q?KfoO7_85TXKQ~-%a*MDowRJj1TD-y zwRV`&vKd;NO3T(-`!|K^R0(Kpf`ut9+h&563Kz+BSQp+8t$N(}f&?u^NxP{w2n#FK z!D-nj8&1NED;{>%2IN@wE+WH)U~W#E0XH+sYSW;&F!>=h>2NWgXuB&H5@UG_OiWsO zLLI`8iCmM!M~ynip6e-(7-&~wk&R4!jh6HV`Pj8FR#>@|V5yl?Ix5ZEpLQ08TNTg* z0!QN{BnC|)-a|;FMVUgLxMQOxYZp(FVx{1vcX7e^@ESd|Y*ricN}iazY75%`S^FM# zm`fN^8LLXS=PEJwa-)t0@!k~s7M#zmW_E<63T*pdFxlk@D0Xth8oEdM5<@Hz(6LZZ zPkZ47^*IA%jc9;{f?mLvCZNSX<^32gLXSeVBFZLz)ls1XK&-D^W!h860BOR6cH*5a zM;IZYZ(;tE^2;Q})U(|K-Po?lg}%Y>h5!5d#&}JA`NQtjB=X5B+1Md8(}pfZLf65} z+U|o^MPMLn6LP#ReL&3(O_8b(7!NGUae z&!!~@po;*g2GOIg#yx9)nzq1czG%hd$Y#8+ZXU*bL_`zf#VazAAXF3Kt7*}uP^a|@ zPQWDy5FS*E$#o4Z3!NHD)f4GLC`vXk)tJi0r4~ita{e$gAx!fK;l7iv&!yBNK|d)q zpC74P^sRm*g_bN&Wr=*GM!mpFEu$U%Ij2ADs8>QOl5CvPpVKTjR%A+OCH`>jHSQIP z+f`-7j=a<{nXg{wrtdpJ-^<<|)fFd)krr3(PJdMtohCe*7Yhs%RB~*p(|@dwiAe%{ zPKD8>luAQK1g#fk3JB2ceA~erjyg8fx-=C2f=>1?%;7`<1HNYWEqsRyG4B|4(7xc) zVG7s6J~$>DFL`PwEo|_HY#8#|1=1x<7vj~J4q|liR>|0ana(78!7v-WAe==gY;aaI zQA0}*-AG&sWNab2h^hjwH2x+#haM!Zq=6X#!F#;a>zp>i7Z`*Ejlua$D1_Bs(94vS z#_SGROHN0Fnhu2AA(aDL3mdrwKdI4S)8>NgGg0-39o)KBm>~IE+Qg!3%LyCS6fAhY zPAtj@QnEs7rDTQNIawjX#|xgqt5F`jhG0&l2@SQeXLb&ji}lboA3mv@HqVPatx3A7 z1r)J*#3{wO*PcZy?o@09U!dE?fHrSO7XwTvXG6Nt(kvauLy%jI2w9403Nq@j%;yR3 zl!Sv?gmnrnxJyYmxJyYm>Ukj-jIfr(Fw8!MMe}v1scAy$F{enkpjh>!2xlD%{k}Y_ zC2NP#Nna5gMwd|=@;fRi+xNnGZr0mMKR1320Kvj&>#3+D__ zz3Tauy1rnSJURZ9kF8)uDl-fg>HZ9#9D|+ZlC^+Kai@=mEZ6i=J?TVVU{v0ni6vlt z4ap@-s-2947(i^x3B;eqE)@>RtCODGiyM_r z8m$dhqY}C(-!aBs=>l>}(KfaWVn&j;Qm=aPN-u1+(sVYR4sWG<&3-GZU8;i(RL!CM zAqP@1x~};cByT0sZWwPgLRGT-b(*mLra5> zGH6U4qw&w5$+v5>`X0&Y2jS`vu~X7%L7H1oHKa?gzD1sfz+=j|7-Z%|Id$cf7s>4K z+LZyeSRl%+1yru1;tu-mA%AI->I4L0zBcL5Fp$jZKWSp%5skIQv3`xUW#YmqP0Z## zYfD`?=4i7Mm~m}u3*k2l;=OlcHq#U|07onKmD5GXrqaFF<06Ny#793cu3 zHDJM5T9>4Y+P3rDlrQ5sO7oQrsic!SY0^gX0l># z;18i&dx%|2n5b%3^;$ehgzxsW8+yIYrpi$U@A}}LvZ=zck^|ZVCAMY*Df|r8Gns5V zwt{VoTB;erDJ!p{stPb7m)1c*OH-a}Ld#lo4S;JwYoYKyXn6u4m&wAApwtCVg5I7^ zimM11gOGV<_EPYJ1H`()50O2hl=QyNo*!8($iW1wl0dLtP8xHICBSLL>+11tkdXi; z*g=j68vBIVXWBxQ5Of%cPS`Qy&VZk*GOj32g+tgSnv0l!N)1J90J}Sg4G`DHi73f& zz*ep%xmd`H(vY}ATGVVVNGg*gt4_MIYOE3sz9Xqj4whH#2CBcRFV$lDJa+l9X1~4M zCdrjJN31}aW|ys~G&TY4@wrbvpy32A@O+H~N_)XYslU(5IU^`U)qAY3<1=O-c9-JGrSqZWRdxKcKWlrtmQ%%J)_^BK(&y)RJQ7#NR z4U>{}gLzdb)H|5Kn`QJCX)=E|)&kUb$c74Zyukd?sVAF)yuFzn`2UMlx54@BFELOq z9KIUqYQ3ZtWhUfkqs-zlYO^0Vk!A#a+?fC6+sS6i?$K9d|C!M16&W2Iw?y~;X zCZ2UeA9>I1g^y+p%w~oclh7J5f5~2BhAldLNNwFOJM8)F9rME&GqnulE2)#YB7u?q zH-56>Z7{#y@#TSpMn5Be22f~%Y=CpPr6F!)zMX*(S`HKf^+nvm!2i6M zfbuyd2E8SM|J<3t)-{;G)|Zh9zyipU1D>AG%GnVHe3btM-@s!SvZhU`66lwh0jiK{SkIB3c7z3=}c64yBgi2)E*8ilS(%( zkBEhBX08I?9M*gj7Mu7cYO7=z?IDGjxjb~NJZk!d;2C_51cZM{tVYVr^q{9;_Iuzq<*{lOcsS8ss*_# zD)#>iD`?vNrq$16_tw?^GK1W%o>_0IKrWmWLrp)k-jw@GF8AoKwrxYMCKaRVwp~%+ zUU%o#YE9%G%8p=8-DKSBCC;u@vt-wkgcNzO1T6rCRI_Z?lS-qQ`VNJj9hTTV1n$gh z^+(;!u^M@KSVAS`?}6VI~m$E+7T4KZt=IWM8jl z-dPhUPY4K+U)(Bsgesx+tGFZOg<0G^06ia6FB2#G(wGNZMhWPvNk$hO|?g~s;>*(&=LX!>PFqdnQv(NyZ z&pabm=K`Kk&RW*fnzqhIGTV?!OYsZ-)(|w39^#OpQNyps;#Y7y-dbz3i3Wb99xm`J zsX9wT^}<*dSc0(Br6uCwONW~BQ57> z*2F_fjljS`TL|1D#)Cv>K>cMpV_%N5A(+k#c1u80`wu9;tK=H+0GS;Zzb5Cr?5p#! zL*?*;w_(xCT?O42D^Jx`NR%wRU-quDk`})Zz0r`kiEj#3RMSc*Q)~*L*>xcT2374b%e@=Q z8Vhc6vtS2+@`BlD4p(wI5cL`aQz~s)0+*7Q>LZZbc1#s?A*JIS!JS~zi1JXdjm1Qw z8fCp7Q}=ZmQSp*~t<-YMg7<*q^G4OwJRcQXADJ}R&>r#23=lm@c!>@~WEbDBjr16M z>h6$&MwusTyS3(~E}W zX2u5>5^ut!W4wZywsAmq5iEo?S~UhFl_0C$MCA#`x?n=%5`r3fwZ2Nc5H&C&%l_fi z0?ZRn^w^5+K8lbg(o+-rWu@-bSthTQdGUGnSR;bW&|f6^rpJm$)JBhVj!ySLDd-;y z?KP}$7_4Miekp@HOkJtlVb^cuLXsbmNG^AAIhP2+FIoA9=$z9>x!$bv#3J>gG9Bov z4t&=|vjq~dT2N~6=7s*#2ItUV#jB_>67E>=WTGsAhDfMEg zEUXB=fz`ZCy||&&i=#-rsC)~&qt7Vf%X|4{!=zI^3PNSZLhbhtG=*oN}?dtPZdgba69)sSIXpsVw zrT~c+BbmoSqD5^6B7FyNm$tsq{$j{Vv?vzVBwA$8yb>+SC?uk07nvn+Agl-bQZ1rV ziUH$(A!TqnVGKr;R1#qwivur(HJlY|h?Tb9y3r3_L*~0|W|RCpiZ!G@%8X<-f#3!0 zLuk_;1I0(nj0B=K4!!XQ);M1~?PDsN{o#o!gtf9HR2T>&mGvS@Rdxcyd!L0w3WfwO z182$byT+)Kg~TT=w-%BGTkZKr$XaBcH=7=Hal2S4T= z%XmVS+VmS|;?T9)^s5I|*Yvv)*hS7-nSQ;Kf-pR-TNno{&RD8lreY1%4W`phVJBK> zt1t28hNX=EiR$n$2u=zuH#+`HsAzsMLbe$j2_awCmbW7S{OX(>IzkBd2!qIWkE6TQ151u;WOz}XJH+E_H3zwczBDCWFIQk_HH zk1B?bUT7B4_Am7nO{b*4e^`k{?)Q>=^6uB{<=tMwNN-L-5%6$62t6u@82O)xOMxK(bsmTBmFTgp!IDvgnXC(@uTm#JWiz&$x7hafNg1`ztt6#9YzBtDz zl+OtRw66s3&3~zGzE!iQxzoR|_W9FPMqFy*ItUdg^&8V)ugS0W%K?>PkjZu1+W1W( zb14O$m|!{G_4`(wwyO0O8Eu_J$7UtmVsE92=TRR5OZZfz|Dm+D0dkqTveu_4QjIJP zis-pfgci7|=iBA$mKD=lV~)xjK~z3Y-!zE=Ki+(M@o-*h6e@(=Lyz>YU;Qa%dCt9f zQ5?-mb_rH|E#Kq*6-gj!1~^bwQqy9{l4+ReDrni00#bjd#r*w)qWG#Y@p%7`n$?kT zy&a3DkbeJyCUL?dWPu%>iAHc!ik@lVGBOb_>`LlA#wJ;xN)~%PMDMY1xG46|Mm`dk z675FtKwwSVZu#Y<`IQ+e6snkh(u0MinZ3EEpKE#QWXRt`kfVRSQlq1l3M7)rN&s2&@Dr?5LVz z^@bZxz80%&=GnV)$ey4nM$KTH&1~WsUk8mjMC(oxuIo**0}N>@Nn4x-DIDRV2Cxbt zlGHfT*C+YiZimnrV`)W!=nDcxTDqMm^jLYzB1KxVNv7yZ6de@D^^ZwAP+WpcO)y^E zFUpJ_n8s{CtR5kchOQ6G`quQJWoT*0%~Gw&RLj89V3!$JXQY~TmRt%pXN@X14bY`(ExC89&FsAD6HP~Lw ziD45G+}^?kxeX>R`pKcbO*SULV`%dix}LUi#eDF_^I;aQ`$tjXZNX}GatUc;`#5X0 z@phnQ%l)+f1-W1L@8oJG+k~%i;>6=thFRc4tpv01xRqb($NMR|_8_9JB_yDmxNiX{ zAsGwtNsVWRCiN^Hq<}F!sH6-?z*WjHF+1%qM^Mp6%W%SqUjoZU z;t`yj!B4$Gk)uNTj_9TWES&TPC0EIEY&i-;1?cUJNZJMzCk6=tb0y2cCRNvq3MpBR zYe=slSrnL#?A0dA@jGNFy;G6()IyZZ7n0>LMsGgO7@AK9sHc`J-uba#)kUx636%Vx!kB9uxLs?=d z*~bRSL%8an6&K^pB;NGqncifbHz;J?e}c;IZN3~9Iqu? z0*t#(;=s9#5A^aZv!6qD-^&p9D5$qIRONb$E(O`y;FYRrYc3^WKirEOT&OP7;^jA- zbc8w$%1x8qMW3|Il8X2b%Z%3wvyDB}=I4}@v6GUBt}=yfl&X@wVZK3A2)u-}Q%Qr7 z_^XmKni`{Jg7TLLxln!V>6p@}&wp>2ucl8eP?Ms&%T3U>N|4r zRr|h-?B zCx4EZb8sVIPBe0D%sIFb#gMuu*T$TK8%g_{IAr6Rpgy<}`DxsfYh%vAjmSysMy`!H z2REXfOK#+vm?Kw^5J!xA)dGjX!t2G6YVS!o96vJ7N41lL%x^^!l@O11Uu{ch;Y&{> zzgNl4>02C8nUb=LAIMbgbiq9IsjK2LWP(lkhE$%)n6RnL8+6Io5ywS83*6+DVt9sk z!$#v5bdxGbn>G&kAXnq5D#+iHW0tq?h8s7mGehYHnz7B;axJmBOr(|nID2z~_ojZ& zC(sSm@?0nH?N)!~qLQVK3GwC6zjrq$bynLt>1Ctpa|)<(u1wG+`4o9FGNO0At(pG8 z#3gQz>g?fdtAY$|z3g?EIOyM-xZu6Kn+xhDb`QEjFmoZekjq(-XM|_IN$1R_AyZS@ zJsb~bpj#L<@7}yC?*^URxO-FDD~0HZl?JYS+3lvzC}ojKZ&WtBsjmq91h^q4z7{k2 zn6OoRk+h~K6qW5<-uq^_nn%r!qGJld{(kMb9+78O^6Ge(Y_pzV&TMlcxD0c>W{8|= zjEbxB{0UI{iS2Sj1|8a~p87$l>opmyBzL| z+^1E-i5!Y{pr>d{x8xiON((y9#a(t{Tph?Wc59L(5pb!X=pRBRbPpg#5_4RCGSh$; z>w^eouKHut)Vk_It}}*MB%3>) z{4AUsH=s-(Pd*v<=efVtrYD8vDEFB6EJC_-{(Ht<`4#P;$wV&KAcZ8lQ|JM!XIG>- z7TG}U1m!T@7%SEYbGH0!zR0owg)sTe?jrIK!D=8XB;bk!iG{(th&xobA)xj?KvDo7Pq1pZ4r$$rex}nXpv``A z6Z^jDUrY@O+Zq;z$_DF*n1pzJB!!LkXo?)4Xeeq?fci)|dT2g=b@}jzLvzuv0@!nm zZS+qr5gF4BF((tw%khWdccpAd%mbj4LpXhl8SK>OxGR0HZh%QGhxdej33a31KrT}H zvZ9Y9#9!G*$KdK4m_u4_xh8J8W^S4Kt#G2%{eC@>rRlS&yJVqmo66NxUN;5tzWNaQ z_VNmyg)pC6LTMY+0D;D+6^8d-`o1EV#{XcwjUVS6s9r2O>z>Q&O%KV~$sEb$oe{(j zr4l>A*alzH>}x3P;hFW6;RecHLtD4wa-LI30BRo+qy znSjZt-aUeKYVGx^hyNRxzY^JEHdL}SIIfc5OZn%|&hGSQ|2kBPp)1}2+h{sHgkPLs z5|42@MU1}$g_KU#2dAk%V`u4MF#W8&bZX_?4C#XC9*7;6o1I9?2S~Ak%R!hC*=?fm zDEH}vy2*}5bqQ%0E5&VD#vr`u64X=P-E36S_Kn&4?ODz%3>z}*+^A#FGNF)8;&9)a z4I=6*_>uOpIG2u)3W6y#`yB^104PpgQ`%xlccSrx66Ff96vmSivS6L@>Y?4~Kk?`o zDf8R|Sm(189mt(>lhY|B)mKxZz{K4`DdU{BjDFBzjFxqWF)696Owby*ykY0GnWi)9 z++(}Ss8!o=zIpod!Dlo%m1rkK_;Jm86GpufKrFZxKBk72T7|UlK}xcl!5*auwW0ki@CkpdG)}*Z3{bUbO@~gDCdWz=$1!R8~nFT1IT+ z0V#4cCNdkViC2G-|9#-QKe+c6-|_jqM=wu*xW)Wefj{r6aUZnk2`$hJ=-nMgU~XPB z$1coA9qH1mkm9}-1SZ1nZG3|z_Es)f-ieH;UMA0$-@P&3-4(xH!v(}=)GG2SE?$MN z+MWJRy?Rc}IX)y0fa>$7=1R-2oYnm47M!}EiX63kEUwarR$B2|{kyMf{IVo*}G(=}crVN5z#bjGS-EwECrv{?;_8XX-wDgVZ&R z>E%jGteabH%;f|3l)YDLYOjzj=j{@cPv1*m~)sOSp ztHhaDXXmqje{q%bjOoxSA15;8I2%OmeXG2B+PpJm695IYM0H~K%cUZAR_zn}S8X~F z4*gcqtN{-p2W{wV^A`RK;3;9Kuf00EcoJ+w@e2+@3smQ#cnCv30Li%q79 zG~_;HTTIF4qAxwRyUr(aFTPNx?(s=y(jMhb0dyqNX1M3rXU`$)bY(u#fY22#`&*X) zj9$NerZb!vIp6{o422tiW25>r2tt9ds>?;>V$EY!I7Gz2RTL_f6?5W{c#g!Q=sM{{ z>Zbs-k53^WYPKc)B>}gZa?2~7tTo!@e|CRQu`h7d*Ssm_mqRE<(@D#%qyL4j5V|`=Ilw4lnkUL!5M+x0R-A!RRz$)5%zhzK~7eW zdm(&O-dj+~p}`rz*Rp=#b>N^2Q%5pPI-^Ef+`jJNVrjd#sv*c&$dq)ab|JH1Kiix; z^ZosNQk&yi&n0tfeNZ~fS?LYoMJ9b47=@7Yifo5gpxJp4Y*2;K6DBGu!QCmSR?p(S zTd>6!eMaj_ypEeG2Z(dQ4+<4#?Q|{Qf8QZAOrWSkDKoN%Opl%jN!-xwX9s+2nkFxlPKgg z(^&asT${|F`IZ2|a=X^InSK(L;D(Fy38*a1Z{MdCgs&H5Pnil@C0dt%m2_?M0&9Xj zv5Lorlp_34imPVotV`?9DSX{7hbWKK1O;q$6^2wBX~*AJtU8dK&#)PNhYN?h@X^3l zA1*Ey>QhgBZj&Yn$fL##=P2ExoXqg~Y$Mj0#Y4sjNU29+3l2O&=tCqI2T0qgZ&Kvm zRGLqU>x)KH({(sBZ?l_fiC1FlnwOcHM*OC05)Qdl*gcK=Gdc^s`OKMA1AON=3%Pt@ zcl14n8098-(_eTuiS7^fGD?RXP7Hif?1_{aO$=! z=K)a*IzI{kPPhPmA99cUVTis%?KPa;^Fn0qT*dJhv}RL%X|z-cjkfQ?4q;Tti1K)-Fw0<}aA>rhBj8+0{KhKA1zD_Q zU19?RxKQVUGe2e^ecllxB6g;M@t?O{@Tj(s7vkC8jKnD4ATCV53bd;EI&(T7YQGpd zm*UK`msslX2sxovtwV6f;;XYohJq+|~@&=4g>x`!b3R zJm*AOU~lNfsCt@h!aL-P@N8otl2{(2s%KjdfrTb8ZOZb@t|>2)XWM?D*r-*{wm3}6 zUB;}Wv^AdXg~Bo0{Wfiln`E|;%rNX-(4NLVHjQC5lcGSc85;y?E*r8x!EDxBZtB@o zFxebhm(4e>AcuK2omoG&Z@C(?S(g~sC74a83>a!Q^)}6>juXQlq&S;qZtGux+gha2 zLs6m&ur}qXuwc+k*uWF^7hz&PQ%#|7-80qx+`_~@TI9_EONd&79f^A;0-%LM(Xd2I zCSgs<8>pBwaMA~+u~|}1CVr*3W>Vo-pER}pEo_d{e;P7NrG<)lsdGBj!f!r=O;*U! zLztntI!qe;t^T!}`^`>spb@OfVL;+oihNSyFG#RN0dje%A+YQycL*#ebV1;eOrWwo z?v&;QOUM6h#g_aa92F*_7XOsxgaV8c5*b!zeoW_nHkn%{ByW-TAJT6)>%O;g!|^JY zeADM#FyfC`vZ&MI$fA!|eF>1Gmnzx(kX^YEhMw*r3;{PJVaaxy9?c7U4B3B{LlfwL zDS`@#s6pmM)L;?2GXyykHS-yoNGz-nQ4^giBDIGJSpPl|QL(T%T~RBZDlH2uVh0|} z+(b=XYXMmBC03WK+Pv12jVGV} zp-?8_36eAlLsBj{GAv1DC0budWW?>kp@f8>Z^*U13fx6akww0QMj~xtXsy_1l7jYA zb{RjCDuX`LeWXrkONx@igC(k%3~=?MWPm%zOJN)(>P}1aPu6@X0r-~KWK`us7h#E2 zBtfmG19b&9ECwl!Ipxb>LFBhZyVHo);R+fQ6;v7rl?8FW3dfd-w*ImjhW+Y-lxnS% zIyd%CIk9r|@>FeHP6yS?XoV2j*p!g&dhO0@BB1+KS&O0XU283(%g6R8U?dNRuf4L5NYMKpndw^ubzeGu^bAS$9MgKIYnBtwto(V#fs;ee^#D*TAj_bi(!ABCdnQF9J zrdcN*`y|6Y7cXy-haI!d@f%P>M2@mq1}JDi3Ma3okr2=KrAT%LJ^B4wKN4-}+&^QD z<`jiQ%yM0@JuPgMy^HUpw-Tw>ravF1p5Gf{ zZDFmwOPyp*(rZ5Fb2H=i1X43kj-sh!K;({H@#+?TSU!$aN18^e1+L|Jx^>|h!Wx?@I;45>>`Us1cTscU&Trh6gA{*i^yu5dzdQd2#8nX>Ba2z7RjNjgC6~7DpF6#G@9GE94zdfx;H%A8$Dda(9#xTAg z)tXWsp-6ZhAmU8W01P?Fps^yOz)9yT2ymg%!2&L5P+~bSJ;Z`;3C3WpOrlXe0MK64 z0U}GTMO73)L>TeUtAWTg(f~#R3*BLIb#AV*qtR39#}s)O?G!m>0D2S^Y%x%`s0D-& z)u8|Njsl?*W%O|+V$sMVr$r2eDUjz7Kn^Z=d@pDmYFacU5XA#w`x4q`73U`N#(S9q^X?q>z04Fx zN9B5%nYz;iW!pJq7)-{(9MUM}#h5_{hVuFXxT!hUBEod-2t>XsbHN~uZ~^5~XOmZ7 z5wo%zVBfMcfqAIeF=d&P4pq^-@iS)7VM3E(iHWc}P%6mPw!hl1F{WTE+K#ZUQ;ukJ{ zq%1!2(0rKUQNWP2LuvS7a#2LMKSIj>)Eyt4l$$8e3=b@%7YdR*o8iljh~|b9d@-ANYHbCh*8Mu$>a&*B*LWPMS0%7LBUu;7|IxB zN^XoWGMCH8{w^{OiFDq1QtgzvUK;#0Hi;VrHi=P*IkniNC#;ADn^-(jZW?TQ+*U*o zcY?mf1=Q;q9q4+l2OJYk*Y&K#ELt*(&#%`uy?2opsaIXWGUd4dOSdj5s)=g-@H#lK zO!AtVt2Qp7l*KAx%_@YGI5$p^oygL||3lIj9 zHU5FWm>EbKQ2a{GjAw!szk~e144u@Zd^Hf?e>;fRvxybH5aRWZH}dOhGOY8|lX;^$ z6q4#)^YDdYfcE-=D%pu3$D9t=Mtp_Y-hs%+~3=K{cxEOGsDKKNZZZH4|(p1&3tHIi57!-XHpJMQ09GAJerV zGZ^zkoM{3P{e+1)z<{2^%h$>ivzL~G*>!*`*ml6jOyH%wyiT$!+yIF{gpeP3DeoWj z?WJ^5VHq!h86t?_)&t^>*o9N6PcJP`y1;P?s0C;x%uAX?)iYtF1a>%@IHo2FHI48& zU#fmoFZ+IpFuL^eh+eMw<=VcNhxKycmpB=gY97+db-!HS_j0P2irpt-ecwyY709X$ zzl85BHM^ykBflhz?b1sENwR9}m*age@i=GLuk_8yzL(@K$f`}h+}!taq?cQMxwY>l zkuq7e?U&p8UZVUd^jhB}7t&I15#YckCYG>U3LG-G)WlxrY;D2;Xrcs-Clg1zuC|_0 zl_)jomPDyV{spWq5v9gZ-pE6Sdlwf#@)|CHO=1$DB}%OX{4eLi6nz61rtwxTO!duN zKnE;=LYW;d;8w{69F)SsVv<15f4lWgLEjGPK#9<3g>OebEx<_hbi4nDxUQSL((nAZY3vExQz&CltCir zTkC|!HSM_^@{5IJ`iGK<4ur}m<*(k>k3Uq?@5XWprRGHW^6CqUd?FUaVvzK_LTwGJ zXCysUH}Mk#(9{0$m?_=QhSkTwaM%=zjk|^J(Pu!~q=l9l%8b$Dfnj%{LpFSj`^&Q9!5Vtsb_u<&Cl!-N0-|^ZGg&hX zU;~B>=cyAalOeXR<`}J_iCMZX@{~bc$5Qc$&cgqmLB7@Lc4EOQ;%(0|v(5U|D`O>n zHjlia!1+}TOUCkJHwGul$v#HSDHEe_1QX`eH2vxa6NU@b2+bU=W2ULq6D&;%36#|v zP3tm4*#83F&M31MI41%_}hN(6sWQfoa3|7C5$c74=&1`E%Hq;2TVX&|9&HkkxnVp?P@@r-o3ud}|5n(BTpbw726cXFrX zwON}nvEbb2t#)X!w?zAWG_*X1mJ&*6TCfy&A|xwpf@_!^n21)a$%Bp)zZF}8LKcC5 z|GzB(Unm1^g?u7=1=FP=0GC*^1{6%!)6g-Ne_=KGrXxaNkbfWj_9`%PKS%nl3Ni#Jk}`5U@nNrB5M~n0`Hfx-7m9(=uHSEeDB-wAjy|E^oL8 zgAnUPmp4q8M-0Lnu@>nvM^vw-%LPnRz8)MH2qo`+z*{o0KuPJkujKmVFJAukYVXmH zUw-|`4iSZBKx-F?s0rvO#N%jFjbr&@cy10h$6ecC9fT{ohDaq=fS@I-HgwIi(a5KE zyWpGRC7b2lQR28yId*3eRUiOxW`KC?=bV8V+K4Wu zIn_UUJiJh@nZNK$mby8_zUKJaqYB4I@`mFR_!b=hoG?qirs<#69RD0|8~u5BFvs5j zA~s-4W*@l+>a1pDoi)KeQveU#^YFqLx01@Bj7doM;n7u3&F@(uOn&m}h;tC&K`cetbgR0~-C=?X&aCSvV!_t`P zHSpxyaC9L9JfE}T$&&s(7vkfz4lRIrc&s=@`alk9|#M@S0RAT6?{%y`*R_7 zel>D!hV~_L?Z1Bwt_>3OIt1!DvTc(Z&{p*8Mbf3zr?#i&)m@D4ecr7jJ-AaWHTTrkoRn z2sBE9X)9r;sV6Ipuq5@x0j+2ZfhZl@>0)V#!xBp+qAVhcM(}*a(swSn>JydpI1n({ z%!^T|WP#%0sh+Z7aajGDufzu_#_VJIG5uaiWV1}GAg`YA`)O?SnIToRHh?l4{}n#y z_E{S+kA9MLL@0txSa`62foLn)1{17AN|-(WiYA&-gIDag-1j-(046|W{G*F&A`e+c z;?i#y^Iq$}S08A+t8x~N)v&PDusAvQBN-wkIoeLwlXxzw7Dnby{gB&bWL9Joy~DgL z3qVo5Qt~^21(D>d@Sa!FPVo%;72mLFmfzhT$(@f=UYVKj{U}%_l~NGCB4<(viY&;= zDqjblG#1<5ri{E4OFoFkUb*N5V*mv9vm3a1zhKWiz?hr0Mr`$M$%u`}eH-9*$7hU+ zn>3gV*x8D$8AU*??Z%3a&1*zkuxZO+yUM1GR4>>ChLWhcj$y+R$Cp8uh)c1R8A(^D z0d6*=^IZp`$=g?AiQ{ag!30X!wtN7V5Oxj*EMZAg|6nX}2yO>Ue(X1X{5P_|lF2=R zxJ&KcVh_IdFilt;B)1UmFGf)+j$j zVqiOQAt9*46^>f|b9%oLW44r0Zyg4O?GM8cVf7ZJa5x{vF$R#yIA@2ssIbfuu=oGe znlwcqe|ipWvM!0mg{VtKq)P2HolqQyd@Ilt%$(9gqaWD4*}?U_r)$En<_=oa@b_bz z8@JFA@eFB0`@WB{z|Y(74|c2beBW82P{3_Vw+GxSkibcT-0}Ydi9H3;Ww`a%W?O(_ z0%aS+&;XDK6;IZ2@-sw88_>&@8#E*BlUbs&Qx=)?kqz55U` z##QfybYpC!4#RF^s5~~8vtDpoM=cP!}oH(eh>a3oEg=1>>8+kU(t-c zarLadc0U)ML!i0?IjD|1j*-&R(S`hut<>$+kdNTTCFCR0kzLsEhjRP%z=iXZg(~dY zx|@av1QXGHI>tcz8sG+_gG$gXPFDR9B#mmTdQ8{atLPC9HGoV5uuFp>zG<_d%ZOGw zi%kqUx;y}q*XZ)9^->P$a)PF}a41&p={hMkUmA;3I%@P zFbc4K$ldLo6~^%)o*@2yPIzIs~hf}6F;cD=OCO^f}ML<2At}?UVV*{WE2BA$}Kw9gkO+r|ZcDOU+lQ(a&t11da2 zc4*b6iX;P`o9u0h>}^5TwrOapNitEXK0p&#&zT6a1dd+sWI!7g_C+L@^=j8Ce4Jcf zn^1HvxIV-jUAGHuBTl5+VenH8o;EiWg*-p<(EO;QdDqLML?4r8gyE4RkPfpl$XsGbVC?WecXfodh*#U*XH@#5x!1{7}5;QHh?^Ou>dZ2o_K07Yzv5e*g zqh;&@|9YUgq6$EBC`!3uG(X-T`SAwHMHKnqw!TB;&XzVdNp?*AgMjWmS>R#V`#5pk z_A@9ZqueG#jZF%yqT?_uMWfV+Do?6I%m_HsF|uHW=JUAL@-c7cKQ4*#Flk=qhlqi# zPFNy<$`=YRgGpULlcS^&RUXgz@p}8!aSB+%RBkZ_HWJ`Q8XaGtF|o8`CHe4=wUz0C z9!FtSc3%mq8g-f;m37m8ODNJ+-g27T>bJn4WTwhv!N_ED@nUtG_6&21s=!sQ&5onx z)VrccLSz#iFDA3&>LALe)@MF_k5YE&yeR{ZZABb1&1V)#j6UKhM2l0<*lOG240^C^!0Xj9tqbsqH^bfOC=+Z+SLFCZeO9s6)(0rtpm_ zD<>C_h$f;`Opf8 zo|4S^<-GVvdE_JWqYurGnZ)+XV;_dP9^RcF7Sn@<)dcoXBCrz(EaX=k-;W|BjV?f9 zqb>{Q!!Y+mGqo!Qjzhj)p;1sIaXt?rc6bKGGsrkGlmTWFRw#XN#| zul-+Lis?vQZ$mvS2@?}nc|;VGBXV^sin)C?ig{QiTcUSF{7pnp5=fU~f(=Z~BE_7P z+c4#}GChdOD%_=*hea_{(}$a;JrXG9Ro+4|SHA@YrkF(J2;eIz=3&5f2#R@Rfnqws zdB}_L?1(7l;T05fp^W${!-V4>P85^KPfd-t1RW`lv=p;o+^Ae#42&g5=z(VDK(Yx< zYRIO+qAeA^fplWts!8V$nRG&}gmTv+m>bN&fz*=DPs)AN5_#-Prj}QZwhkR7!%Tq~ zP#4ru3?s5IvDs7y=)_|5)4e^3&mL@E?L9&2leOwYznN*&s-g_~pH9 z=G*V`E&FJ0$=~B!6yaO)zvf$FL2t<)_AOHXE&2EO7P@;&{)lgp`)|p=*SApOTk=PJ z%jU*g^6&R8GU_e)W4~EQMEJg)3jxC4%7t*@w{d~@zLg6I^%gF;e&5W+VaCvJMPo7{3Nyw(sxV`A2`bE( zU1th21_u;oj7X_4W2l6}jMHTlkbxuW{6Vzch2r(F7Ot0GeY{98)Yl#eT{ zEp`1Jc11CfF+z2Y!@Vw1NUzU z*uVA8%|q%H3zFsE1LVSK!~H!zoKIPMwfj%6sxO|=a#Z!V&DYp!cT$FD*q(e1-+0js zkh`@@_%4cz-k2zaIG3pr1_U^#PYBwwXcA~d9IkfkJxoNi45%5Q*m7_2slirJWO*^> z5dm3ytZCP;pwqN0Aw@`uOYCR#>u2~iQNaf!idI*?V|Qz1Fm z;2y_&|I^*A?X}Th)c$YXE~r1)dvJHFw?5n$S9^bL@5zTqcSVl+ccojx^nW1Trnp%v zcnt*rEhYJSowaT}15+=QX-1!4vHBOvpf1=^RXwbaNh9hmrlvP)jOJe4F8qu4?>+p$ zzApU9`}ZCN)+R(>p)Nf8{=E;X3kf&B5>VLtKtKV5#t;4e-WBv9khJ>YxIJickk?n^ zhy1_S!jC9}ENJ{NXNYxi;{FuyXgDOIAHk0G2c5W$m8(c`E<-qhQy)Q^InbV2BmnXt zO5c1p2s}%P)zU>u#T?X#(Ai>9?}r`+-pd9}DUo43m9b$LQew8?Z80)#LNKt)WTH}> zX8d!IN3QFy~FXLFG7yx0^!c&_{<_Xwh0_glgP1b;3OfgFw=KP?49$?lW}%p zIT2SFg)A|GMU7D~lrIW-LV%_ZeV)qd!HcoPuC21T&*}z8=m@Ryw9jirQ8ttdO4~8{ zq0uY|Uhs{Qu813>+Q^opRFf&JHb?G2cG;c2%Tp}aC8{sAL&DSe)0)x*c26W~Q(%m< z>eifI#~}@XIqVO%+2!~GXnsFBF!&!CQkg+qCDhcZXp2w)uc*j4TA?V>^S90rP${osvY;bh7Q8#&&DwYFdaF7uCML)~y)UlG+CkT&?zrSFr;6R{N+8pE^+OOJzrJ7U8P)XKB$cs(pxp z|8A&#@TnS9iByMzEwv9ln1z0z+UGcXPG9b*eX_SNs;Doj+K1k69dAMH!_Q&&M^USN z8%w&=b5#4REiS2j@3mxsuJvV7`Adw=^R|up(h=AnV5|?O2z)g#t)pOVOhOM?Y)TZpv_J*9x=B4FPHO414 z<|DPRb&KWI8lRFTr%f$Mn%_@-7Rn&Jz%n@) zPsK0Gt0Q{GxP3_YX|3VhWc=c5+ zi~pQ9grNg{3UwB4-Z*YiE8dFZRz`rZGWKsqA;R{L#d^T%!5~O}Z*AdYE1-Y>2Lv;* zUHy7#+v`^_65jD2;>njr{}8JGTe*W%$Zkp9Gd@-z9UUNarJdePQE_Y4f1w5j=d&xH z`Q<%Y;^v=mqfBO3bo%9``zkhRM#JCPG+emhb<$f`SID?Vu2eFWeY&`KAJvrYevm#= zFa$M2s+6mJ;=by;h`-F;`v;qk*y`#Al}39_4YTU}rEPulu(G3lHoJJAz6Yb#*e^G= z<==dZY))eIh^@WpPl>k!sNnpfmdLUIzP9B`Sq)OPU_K~nTP|XNtoRrYxC3S@ZMn*L zh$Jyc>L#$JQy!abxwy)bEtfJ81zN5x+jFVpojPc=p~gw}9zS_UF(5DF$$AUy2ZtBt zlO3c|rfpKL-Co?qrGGoNAv|KQ$i4;L5rv!hfr=xq6py8!M4Ni!tIot3E6S?>YPCQr z+xL)JYEu1V#CIKOZ;x{OEb26siDB`rWG~`?&SzV*m$1<|pUj+!V@kRwz&V^+x$H8&$?yD9@DT?tF%^h{f0qsLXRQP zv$8xBR+X9b;}I_}e>>+%D*%lOuba)w)7mWR)Oh)9&Zoz!!L{@o*^~~Ibcft{hZ56v zKgx=@g{&-HwLHW6aDbfdit0T3v}uQKDR#o^j)<>ZU*DabF3+ehGxvRVgGYH<-#Hy` zHJaH@sSeN2Qfx=fWo6vf&0VL3ACEf08?2&nc>~Sp%hUp!fnz-YKt;T2KD%B()$kR@ z2oG$>2P$f)fY@W4%CvDhZ5KI>#b%p+O=mOhFFq~2W#bL0bj-6umwM@;| zlqEK>h6wQF9O$U0dV3?XvkC!InO z?CXe(A@-e4i3(Pq_MU)A^<;0i%J}tp+)c6k`O8C)p8TT4^kIR&29d5Zhu%-;$a6zU zl81jwpg`aq@G{<=ZSjtkHL0G4!r+Ds`f@_?slugrj%zTUH<|q7IX=dC&M}3N;eS$2 zZtUHEd+}CY(7;a);`%=JtBbPnm+wsfwf7bru>qFOzHi7>I%@_$o*c|(8o@^i2N9a( zXdpL$J2RqCu$9olZ_>i|Y0!*uQoVdv1*#|C{|||%ez#FkQL(h}a{YN#U8MrF(?{!S z{%E0(a&VYF1`f3+%|#GEcg|FZZn_3g-CqL?>#obQ<+aytlqt{p)4Q@OF0bBlQtsRa zM7WB=1w0w^75NtEbAsyh3xcN@@FJoyt>=1=45HT~{rH#^(9ZdZijwUX0-*Z}QG`OK z5T}De2!S>sPg@W!2*NUj*x_ZDZmpsaQO0!36rwC!3c(V%UQ2oz8mz6;nwc}R(@=tm zj4qQKb$26-_cR0=Eb3$~(DU-u`McjN8pjGF2rokLOyrcUK(vil>&7Fk6Wex8@fHD@ zqKRV}U7yJJ^(N3av5d}&d|xjx7zK8MeBT5LoK^cf<@Ni>_d*%*MRBp4peIskW*Yg~ z@@zvNfdu`&I#W&@1;7QlLtwd1Rj$`%I6JFLAJ}oplyc96;z<@&`L(z89@h11 zZtJ}V@?U+@dpBze_{l6$KxXP=Dn?qskLr3o>q6JFxAmUTl_m2bU0F^a)D_fvpRR!Q zab4j;kLh~)w%((7Vhf!93C zZ~Rg|R~7c^J9akEE(oyKtKq37WJE3QP#5>uJJ&#BPv+egz8)oTm<1NhO5Q?;l?era z3$e}a-qX9icSTr3Cm^tsSLe6)p0sKSTtDz1OImopcFMYK`-02r>9_4xC)t$52prJI z!H@5p&G`1;`AxP<_Bh*vSBOH@KY5#ccq|hN{wF}EI)ZEiH6!?p(x+IZdr#xhvYRi& z0;)|vB&84Pca2k`Wa4_%;Y)>Z=v%t91fN>{W z>ZfZka-KtgaYtZ03mG*S*UGc7cckYUj9?E10pn?bu@6iIMk=*n)TCOvqFR8FY4NaU zOnSm{Ccu=7(TFQEM(sRl20jylQP?@Oc>2KXv_akgoGJ1`47CnRI)xA2eG)kj08-U0 z)e}Nt_FRL8YH2=GI#2E%ob4DZN-YGqV)eAs1D~p%T}Tgn3_?r4MJ<>r3zovG3qXK9 zM>>_Zlwu^VLqT89#x7*mrg##yqB|Omus~!p6F#8Mao0>UD+3xq7+jCL6PIQ;!2r$_ zS}>Qt9om1B4Jl`Npaq=`{bVo}Qr+7ORFgicMIa@r zEU%Xjd|p}=Eq5RUpdT`ODF~~G(1uNcPBnUc^`UO6(IvpJoMxk_D(F4K!1rA5(cZjJ#wh*_ zp*7BArRSOTeyULXWx%t7iwQ?diB+OPyjcg#p_5qFGGWOUwIeA%l(g1*Cdpc7ScKb* zW;!b*Yn>7PS=O3`taX~Vo(OCJn#l!Kd7%v0ALtGoY&S+uk8jmHlw3>m(`uumic6f0 zf6Va>TDx2>$XJLbD9r;Fb%W-b8K1^!PjDWlSp8x!O?#;?K;;H7VScup$%=87sw@~8 zDJJK$Z(*Rb@~kq`73EFP_5?TeyS)BF_0C;O20NnQHiR)b?ov`Am`gb6Y)8Znn-Tqx z%38{NBcMVC_st*of7Sn;ZlduBj1{pp2g zFm~ynz!zxnG5T|L7}t-^6!)phRL2STm zyjrVYLXx)CvOKHAUz6%F^!=JI-2??G&t91RbusyX6@y>_KE}c{R8%8%S@8B!TGel; z$_?uAkXZwB1CUx2-kTwVKWo`hu$XYzN0I263bAZa*9H8fm=77g~w-=8lL%<4y z05zw38j1`O%f0oqlx0$;-Pw%J-eAsibXLZ|4>BM?;=+8uM6WO(n4VI%Y>W@+)MA2l zMh{UeQPe@F2ofHa%C-myn7wX(4yz6Lw75uAw>BDJxY%zrAarT52Ii>cCj|I@R?De6 zS3Uj#uJx|*<#N=7C(^ekT&iA>fUNqMpZxCM0LIXO8Bd;CcoIXjZNmz{vXub#kj>8V zAFAiJVQ!2f2aM@Xmrw$66Ue;5OBrLzQp6Es6-2YP$CNj+B+7IAX{<75Ky|k~Guh9f zpCTD&Kn!>YGUa{~he%oX`)AZaO&d#aA0tS1)hJ^vR}{rIQ@SXsVw)+o-HR0_(=dZH z3ODvsdbc$r+P-5bEyGYMW!gNcjIJ!0K^DqbEbb`W00nenWK0Z#MRoLqC>E5YP}*`soFvfNnRVRV^t&DMr?Zg-1G)i3osfYtOiH*w!e0YL0Pzf$*G| z+v1#PvZ&u8O-_2Ai1$t+-agiQ#q&gRcNSX^E5mJ*BeBDCK@2PjVgbJ6R8Xc&k_CPO zP5SKBipu}BFMi|gSe=6s3UmAx;G=o+brJ#fMKK;@&KjIyt+!JP$T;bDrjKHmFAnbH7lXMLP>4d3ioeU^m-HE~^Ik zNVBKQ)6Ha6=&t2lEZL@JvH9y2J9S16;nL4xebR(>l-sy0mP%cmM5jc0El_A3fO3zH zL!{XI^j}5AeA0#kbf@onkYCno_;K%U^&_9D-+b@^-h5EoU<^k^hKy0lPku4-z_PbY@fbPAzmV)#F`K zbUc_c@zb8y8z|%r^4sNcQ6cSLN3fTH3dVv%$P>>qt%>;=oH&SoIYbggfKTGElRSj% z?w#j=@3;^FZRr9oIEzZ#6vhp;Lw7v(owPh`!2_1h9}-i2qJ@NV_W_IP{nfh#3m|NC z$L?sGu}H-dQo~tfOQeZAN`xGKU_hi_xjTr-S(22-i!QWkp2}vP%Kh_nEP7X_c_I@* zL2E$sn3TNv_0=JHSjCPAoTlH$?ib3{|H`RK8AMe0?>r#MS*63-1M6>KzyQ@LflhA@ zVXeS@$|7VuX$06t5;>Xk*;QjW){NoUfnzw`6ut(w~L_6C($F`=B*kXOA5dzfL&i0(Ye zeeceb6c;`wTu>9@!CEYH3mwxk8FScw*r*ZGcjrFX3=- zjD}*6WJEg9q`X01f?UZr5Qb6JUXiVFckC6JV!d_f5QVA_4tIC3RP1Hr?;a9tVuZ~T z(u?qV5&EgNvjxvm9C@WZAM(&7MF+vgZzIY)-qw7PFb3ryT2UHgs7G2xDWqTnLxB&X zSfcEF*&Nh&?0xOs;VmC|^rqM4pwWnsEZ(Xk?P>{5GN?(OM#SFOv>-Fi2$>JgCk{Y7 z7>9poRVp%!yL}u$hBV~GeEKI2rDq1&X$ZUrAdAO9v`g$N?X$#sOTb{AGQ4NjzsX0y z7}5#ldtx2hw+5LS8Nj+@614|&pvzGr`!gq=$VFlREX$1@7fxtJM({7#A4QG6r@SFF#6P=G2ca%6vm_c5T%FMl3uY0iw{PhO->kGXw`*oBkQP_zjYJ~yz z_4})dh+rm&Wqe=DO0JXUD*bPBL zIwnDS*I-Bk8qny(`S`i!5AMC(>WJy4X)`mObO8Z_1ZgoyUvGd00|{f=i2^ebA~sAA zbRtAWjfxThH7X!NU^0m0et&E2=jEI_)#-5Ij`|_}Nu6_^XTPq!_IlrI+ul(4uenf5 zthex+nWFD&-BT07Kb#EU2@9z!###MZ6k;=IComgCA~-5GFX0{aQtNsIoc;b-q1kg( zH~h73qlw2rI@-TJisNUOn=d5&uWhYwe7`T-H=1$sz)8x^?tF<4VuutDWRb-nDEQcN zlaob9js}{cA?O^R9IRhnFm#byjgwB`7tuw0v{nR7JUN6WALWvfSU>TkUA*gmpR~OLUvJduZltMKCDj;>Yvmn zC-v{vCr9;BA<;*0(CL#azV6f~cYNKUPcHeoU7wuaxK*E=_dm{Obif_utM%v(xL3(E z5FKy;dWNhb@qOui16o5l;5Z>zkWYYK?@syJ*`2+MPcv~cfVhy{(`VsVibzapB59zb5~I8KpyRUeMGj@qz8t0gBsbPJ`CXg7&d-E<7qb)| zVWq);ZWtKD&MSg9R}wsEW+Zr`p1@}s!7~h=s|X%DXF>5hZ8{>PirvYW5MgEu%|x_@ zeWjz;3j$uTX+BSrPO6X?4PJqY*Ag_eP#Ba28YSsIasd`okODBuva;D`rTdjMdoySm z?%&F9W?`Pg@*C;Is0l>wOv}ZWRAzhRTKO)kZtE@uNLw=75(hOQIXbNxd!!nQ*;5#R z5`-DGVSVmQwGOt_>XRwfXk|DLcNp4U5Mj`Ii6}#QwV0sBxgh)Q>|;WBhHx*Razoxj zLzM2D|17EeXNmjIa2l0vNHXgvuSg>Q*?_L28*+fqI7f7!xgpJ5PC@)<19w9X%zs9u z(tl<_D7Jf5Ejn1riP%65qA1RNz~ZylUmGP*VCT4(h3{+$X2#ALk?V*Yg;b#hVP^Sq zpy#?mY_sjEVir!oTzz3_Ok^-Fu&tP``NT*EYL}=_Tc-w{f!dvra|XnP3Wy61#JLiP z2!am81p#r$>PIK!5L8OYXB2^mTuDKIIOk5tIRhe5~g`Fy?|(JLO_fR36B<8swDc zRHR_JAk^k18){<@w^crWm>ha^P9msHN_% z)>MQ5VlRgE^lbunTw?YFl+YZIvvNR^ooRVC*H*y+xvAJ9nZc4F&MPF>re|BJ2_x%i zC!i9N$p1J;b%wcMLX{|S>3tQkKJHHzb0m4HiPnnB4t1fS3@5YVJqkdm-wip@McNP|Bij?njDaU65tD{o5~*qiK6e($mum;Kf9HYI@7d9g|W0y_qfwAqBL~s z-J&s{Xy!pWYcWGUEDvH*5T!yE765>7g)jyu-GWteW$`&W2Z@j|P@VWA)lyKTd$axEYN2^CPgaoqpv~=KsCevP zFXg(Vb>!>id|(YhYm+FEh#^@Tml)Q9v4E;K9+!!~G~T;aL{-$L5xEi+vt}ZPNz%bq zKrQ_R6M22!J%WcUh7Mec3dhnIMBD_SmjOo-+lG*6sDTNRMp=xdia!mInCaipF{#jT zr&X)gip5XB&v0M18l~K%|1*zEhh@-Tf7}#{AB~?6^^9>C-G?7bqw9vFC_xYR zMw@`S>68iypOq5V-Ie?!tr8#4ZfDjwmwKbk%glq!Y3@$p*^=NQqilI;ho~gEqGiOh z6ptXH1?Nkq=~+wDYf}Z0kv!8*q5dOeKQ>xWB%I1LxN$-Pjder9DCwRh-m^RlMei)~CyxB@`fN>1+h^+J}-eujMBZ znPvN=uYp8JFk1K&)g<$#phb}-QKrb0Qfy*LRLGvBPSs-BHP}pRsJmY%z*j(}J){Jw zj0R0IlzGxsZkdtr$u$!=m1LYDSFUX!3Ebg++u1RPam|OD--?B3T$l$sjK2qcC+j3r z2Wf{OZ5_iZ$;W2B0Nzz-YIIc-$_Adu!)7H=Yv!Al7X>U1+JTLQE%V(meXVmlhd?xDH@{B7Fe{4d1u*{NnxmLw;Tr~@o*>th`W>RHs2V#a%J=c zQ<*er|H!jPTel(X*#O4+W>MCGn9dH+K!mg{%~r<3-B97z)BP7EXfJl+sdFgop&&sI z+Rt3!UbKBtu#frp41yzoPB)N_Mb4uj9%5AWTQ9fzDv*;>+XloGT1J~dL`G>xwrh+I zK{!AU!SXcL56`BscG~*Qi9zW99t#C5MMmc42HZ6yB-Q@aDxt?cmPNMXoZnhxEi&5# zkUnQl$NI=oU7`K3M*E=%!8q!P78jc;v?mAx3&3KR9?+t)pw4IgekE5LkAZo)ChkD<0q(XdPcjYf7}Oqcnl z0tnB{AduOEfJ~%EYpuQH=8-p2`h3K?g&MqI{UmtVsJ5lsBm%T9`)ukV_QJ4sSHXHV2dVsAhNpBbihahWm{a8J(fF()(T&qDoN7?IDaWOnlRkU|X zsmD7NS&S`s$zj-hC|qNmKtW+ebjNs^8Dcf6y;vzsN7hB0QZP4*nKF>ihP#@uWdeq9 zq#?I4MP5R(nMrJOa12+&!yo57(glh5iPnUUtV2kWNY$#bqv0zhc9>a|qf3%Ye*a}W zII&W^G8|L*$cY#xVx#P;O%qySDKKvGh0(@n!K59N@S?y8CVLJ{giJp=$|*}y?7{;m z{4Cf;FYH+j0b87@94|$9wnTVmNJk|*rhh1gkcfFu%Wk&U4oUo;O3C<|?#SdpzUe~G zvKRwTluZ>Yo=jj;tj;i1+q^ryi7rs##_Ot-_%fi1pp-OIgVN;nSp%i4XcB5q1C$8y z0L?&|{r>_=%QM;lrRJ$!>Yy|j%{q)oJtPs=NtnGppxleUkn{;yPtI zyhqn*^#6}EEj6UZO^+yrBD`owNR{Ekf6lq^&7aW9`1P|lSg~B{b9Wo^S6@XfMN7DZdJv)0}ybTh~>2ALCe=kkjC1BTxd(Ryd zzNN#-8=eka`Kv2vXp`kVw9lGqlS#c5UTVHl-Ef8rlB9bX!nlk#y3t>n*_L(yfTIr2G>)D{pfvoP39U-NDy;KK@&JdJbRj{N3Nw z*MhIdKJX!Z-Oktda|E9&c=)_HUE=fBkH7{m$QJo}?=dUIHB8~&pR}(7;ND{I0DQUB zp;yD#*`!*bx=Q%IPHUE{4%jQ#Q<@#bc~Hl~)?P3KTF#DK3i#a6;Pq#?chuRY>m%3( zMeA}aLya&DONCx|21M{z$;NIoDcDGHCf#x-7wY|a zJ`+eP3B#7?C~FKg*)rR>6v-#Iz^Le77|(XFQDOcOz1kts zS{WAydq;^+L66TYG$OW!~_I+nCrK7t%CZU}>4ug~g`Z*z5DR zjXHZry=``InWMt0me||{PzLME6$!9eqmog7?`Yt?Bk94pt_^6`w!&@u4s5GeRkg>fNDXm>=+3_g1?~s#5FJ$6YFHC0EEA6me z`f(izt14!^7hjVROkEY~UPv7gdzoeMLVwLG&c$;h07zJhev2|dpS~-W9 zdaE*$AW&-vv1M^-CzefO)?SV}dpEY6zo4F;kM zrzl`~g+ytul}b*fqVqINhNR|JMwfh#D#*v5Qo#0uwHiF5&~he<%V?Dbk{ek#PC+BM zLX5BQQ0UV`ZZ5jW%wp(mhK<3Y!v}X}C?>7nU+5?Occ*DbEN_Fc%Yf`-#|S<#cDW9e z$;3lOQhsU{W5Y+*K%B7VEetLx2ad_k7S(4@tj*G)iP4kt-iD1vVvJ^d*sq?+X9RSH zPU~1;sI9O3fSV#L$SovBNaZw;<{-6VMAZP+=@d>o8>H$vG2vo6ry>Lr09x6Lp$ic^ ztqtJbbDV_wR7Av+sv0_LXWGgw(Ip`%jBJ*B^0ong8dddeJYb_)AdwTE2JPc{gZ8n2 zU{4+iv5sSkNK1O@J{1M88q!b!3y*sIW!KL#mfpScXdcj2NoFA)`ebD?F> zx+Vxq4gz98p0s~NPiKT7WNol52pJK;^}5=Exo<%Qo3qOxGpj_Vc6|dDGe}gLV$MTe z7UEFK6wBTt1+1*HTMLKcI$nuQQGOi_Xl1EZeK+Wzd`2%P~SCK?>WKVCw=Q+n4-l}Oh zrn&__Mxp1nIn+h184;X>NwC0iM?zazb$}?alk=*~XssrIj0gi3NP$}gS+2dKnX)_k zZ5x5QisDf#Mo)G8PkWM96n(K!v4Bs_)tPb8d%7<4Mas17>zb+?B;+>A z3g)6*EgO=R4D{q;wwiY0NQ*EnJ={inAyfSdbORf*)~U;#;f)P}Iyco%w1ya(wu7)U;Y%5N1zIh+X6j zG9d!c$_BadNiH7&gW5iZ5(b()Q0C$;R$0C4^+?8kM!{)!H|Zh|?pt2uEnn2sV&KA5 zMc&P-Mv-#ItW-dpuWD>D9NsUR3@&%`JSpygIJgKQ^~muvT(;62U#y8%Z!jb4jqa14 zc*Efvy^)GJ_GT_z^|H5&?1e}Vj(I_h@=5ZscnG| z2H0ygWERtQK*1GP-?~~X0u(6@W~uB9-)!upzYJUg8#JYmk?m@O;@38W&LtM%rwU^Vy>@j7C)E}<030IwPQ>Eg9eTIN@Lzsri5 z(5<23F!EUw@X=r~ME6M`!f1&M{?Ke}TfzhO)a>RiZJDGMv_ch|lCiHeyY;_W zOwNob-(qr;JmFhRE-{k$EhhJMGr8m6VscSJIfcabEhbldi^)|??q&U4mhsJ{vqk4z zI;A=XFO`|eV*KR)I*Enp#HhQ5^`v`!!HMw<)4|w zc!3Oo?#6B4MDrBg!x;Y(NkC{&@LlsZTYx?M9xrDc*}vZN;nN=v9L zqqB-Ll)qz1Qub1D7QSW0CZPT6EHOp|of=hG+>$pzH2btB6o@>%3FLLg#~#(Nv*Jt~ z%m9zD07g!;TV85m0di!>ZwNWNS$L9R=Tbb-5<{K9&T zLb}9c&R|&5rOt#ZU@kbbSmMD*Y44;}o>g@1Lt?n4!t3%#v30d%Xf;*nOfP!|YADda z5GTM4HGOnhS7II~p>+$7kakCBc#nhu^5BT+PNT)xFHxa1X)`<+(gP;?G9TFjbi7Av z$-VOj?YT8r&_;o)&%`}BdAJYJ^%;QVO<1_tA+*X9#T*!VxaeIkm0x&w0zBjAEB^g$ z%YFb7waL=sp@a&g=va>fbbO$(W4Ze2Y}FO3GgfycQeXXyUPL4ufU9`su@VbWv`8SI z`Wc~ioLN|K2p5na@Q5$)nqi!|PcT7fr?}$4w7h3(h8{2;RW)K75B#5+&v(DLo}p9% zT9-3--a*S6N?>L6H!5wD1DV(nDoFGEFN49zE`yKlm`1!L0GPIc^hFOC6{5+lYGqlcd2eZvRWB!8XIwrooNU z-=e|)lxeUp)EEsWjKu<>%yEz@g6i5*qO|+nyZdoK#uNc*d16+fSA|_ttZ*hgHFjcl z;#N!@@Pla(2()&Oxh*Kmp#%9U(&~0ae!wvNenkS zRBLWwOJw>&XP z0m)zC#|H#FBj`mR#3J>1wVhfDRn?*a5MX7ZOvmSI-b{IbsUSxxA2wY9onXsw81YrG zusQ4AH9cSmihePOaL2FrsdZMLGxVu&771HbpYlc0{8*pdKswo{LV7q5MHe%VxzEK$ zp9<3&IVO$wY4Fh+`#cAXm8i}}pB~6FK2HUNu@<&hxIIgC0eg$O9+pf@K%5$I)Zkl0 z;Iu3PN0HY60T}5ym4O?t#v*Xu^uJbXSNI3=_tZ;j3Xc&51X zIjOsw8qa}a0n9aC3UQl`Y4KL^EEgh)sgWWip?y+`KMTDIJ3w|CZMjB2+16=m^Jn#o zvrrDAHV-&Zu@+F*m$E&2A#Gkt%jQ+-*k@IGy-E~YmKSprdeDmLVUOU8w(w$cmOd7u z82~b_{d1OzZAP14l#}tySoTb1^uR_Sb%_?Vbc^WlSUy%qOJh1bk7z}GnCU~Uqr>Z& z0INopVq|$G{5g|*bRFK@l2M1h&CIpvz0-8~-eGJ^wGMATr|a+zNa^sv^Resj^nVD` z!h{ZwqB*I<3&*59Pu1bw6$LoPls%i_nEdF9F{S!snX$D!%?Zfv$W7AG)|_8Z+T9#rox8szOwdOX@WW8w|N3Yz$e9`=*3V0G7>=a zMJI!+02xcE#jz5|t-@lmL(7}2R}YJk+#^Y6JTit3;S!3BG2%}k!JyzgYrY4Vi+g|u z3MQCm0E+|@7^2B~OHLZ3AqXC(RS?PJcot}$Sdr0i8$yN_j}u&C*ox8;9LyqD79!G+ z3`IzM<0Yxphwy0!lqJ=ij`U2FyoPislpTzfozkl6;YaKom_?)2ys0B$HY(m6Yy zP_LShn${TRH6;SW2+{HG$;E^q!|^n065_3eh)p7`h(!n>)c5jgMO8F?AkASd)>;Hd zuXRWW`3?~E$~c0VeakvqhnS|OES6va{xVRFMsy5Ql6{uwm?7H}uB{N70mH{a*60!I z?7-NR^PTDqI*HyWj$q;qq568GID)Y^^D-=R6dx$k^N=FUUD-!qt_zw4qu?QP8W0_0 z*i2YCTYxb+EFB6am8dd^rh~e6Oc-QW6HHKdbP)VZU=2X==om)S89FDLS(ft3 zvOqvWymIDFzy-ZVMBV~rQ_ld+k$97kn5e^o+aQE*qpj7NEBGQJF-W2{7v5biB*qLq zfy6?i8jUYUOlM<&R}&?vrIIxodvikKiIRrIDqExR)K+z-ErNd%i8tAVHb7#6T{R`i z@fMon-4i<}76lS(MaPkNQyCTGMJMo=JL-b0f*8P0 zp}BW|l4CbzR+8{m{R&&c9ro1XV8j<76!Jk40?lY8Tg)I2iAWf6pHU>ao)Jf$ijY{^ zZk8LSA^lF@hR(YU6LfG+x^Pcs0REtm&!d)E8W;@;a_3sGt5Gy)>b1& zj&ypm;I*`CL??w>oD*S&=t*!YQ*?1ei8AqvXR4%=+;k;PUS~fP%c`(ps8|uW9vdT0@3xx-ns$>Z=|gIanwztAWZ`qGK{A zF~roTWk=uSe4>_y={mRgvvv(}#^to6o@%BF;EdCfl_FM}x2ez-E$cf+nW6rf9IJ{W z7Xx5o!?&4`?T3y>5IcyGcwn*WZwR@z^b)KDfOF_Z;23E9AsQZ1)C~R<<*X?nZ>e!N zux`=9wONueh-wkk8t85~qFhmaYC_8|>PXnTd4O0K2>_)3Q(?#jM8C)^CZ1nL&voqF z!qwe^GM9)F_t`cI_);%0K9PK&CH;;}y|f{BjunivQKk}xZb9c-bCNcRKQE^cXgMiM zd?3xDvHa%f#gC?s9&A`Y`h~I4vx1YOSFI_dmjH?nEC!e__~d;h`97K#O~ecYg=Ea^ z69Jh+V){Ul1FrlsGZx;y6pRo$*{%e)RLC)v4_gvWIc%{)Ksn&KC8AuW2Ysi60?cJ) zqea^X1ej|lf>RK-c?~UIj4RB7F5x89rjrx`oS`8WZZ1^hz?kfEGs0kv$$opA`f7NJ zy^&g!)8Y$P#_b)=sJx%c(;k*cPChdvK6;)A+^AKm2a6-_5e1iMFu_m@k^E|Rxg}D^ z!BCjI;JW4Ks(h0@{RTOoN>1KDhdnBvFO&vO+)X zQj|rWj|HF10dXBVg)bqUQ<-#H!f?!%lMRqVp>Ca_<_#~cI^aGz`y26Vt+1n zGk%iTpEFvqfYg>nNY8jcEq&w6UU{BaKgW+0|NbeD6tBJND-0y)W1~aGN|Fx_K{Lh= z6`Rlh>p4_xS{;_rM>|xEju?+2TMe1!>qEtVMqYD26P6bi`I1Y9YQ_D)kAtg@io!q8 z@T6`XO6@g29^r?i2oA`xkd_co-;{OD-tMQ!^{f2J_lMbo0G*5%5pOum!}Z|=-R3eN zS?8=oRUiyi-1=wFkkQAA8)|D?kZ`^YE~@)D7M`*DHyVqzR&`FzwKZsvGK`*X{|ega z2-h_+rbD73T`j5&>SMFK7KFhAB~J{{5o`Fo%^ia{r{h4-r+OH=#GFZ1 zAzSwj(wTBZoVd>27396qpKS(dih>P4TAr9iBldrK2ssd`MS2#&>?p~d++6v7Js4uv zFNxi=$RNq1feaNvM;jTil3U2{f>Xjmvqc*q?T{gi)pOmD+JTy~1v5&>PkNff+}`^|h!JD00ot+sAUV|bf+m;KDR?ri0VuUMfMaQ_0YKpmXx~@^tYDjs zL8P0FY}SgK8fsWe>zp~%M7d@PVyrWtoFq-wQwV7cU!(j`=0QfGPG{n8KA8_%$*>bKZCY0Tc1GC?b24C+(7Uu zwWM5|+Tze{(j>-KRs$InINyR!A|*m9v<3-4Fc{iM=~VM9E2xEf5C<&pUsV}qL<0Dr zlX8t=YCxPKwc(O5!aB3=_QlLAoG;K6%4S%x&;}n679GPKqXa3WD8pDCOa#cBSE8#D zYWQ0OfO6RDhQjJV!?diaj|W;-jz$jDSiaD>}MmRkv&?q-fG{H!RiG^21loVB6a@Pa~EPA&?x@Fri zekfu+4McOuq%*b1O{(-5@6PS63rSP0Q3?ZAn1b@fNMp!$f<44N;IwjyA}8x#}k+Lj#Uo1c0EDI_vX1kIUR=cKqA$~peh~Wwck6*t!zcpz!{!3e} zjGsARJWdrY`j364um@Kq(r%VWBQ5EA}-1G{`Mb@+I z2j`3Gl6C~F=fkpgBNSYaU4qub{4btMs3krL8&lc1H2b0M6h*_Ybi?&Dh$o|~22a!)sMMX{`+nlW`5 z;A_*Bk7X+OAUTuZ8v&cu@N@t;0l!T0ScISDMwkXaX~5TU?ea2hA>2EtNYWa1`fks1w%=}&D3QCaj^R;7$O%XtB#z~ z5u535?~E7)UP7<6 zToNtyXHd6iIM148+R^M@kpj`vIKV&7s*8U_4u$-Q-EXF~WSmIC(8FEEQhx{;IJ9Jh z1^o_&vq%ms*7u1-RIwGF=Twm-`1!OBCXO?`n( zw(bSxHLEh=Mn@BoDrws&)#O))|JAn%y=tSF63*_`)P2LSx3pzOo#C%4wOOW_iUK-} zRSwciOG1xkPCYQ9M-yx;=!+H}p+k~plQd`utXi`gEqO;}VR~5CD+k1ovF z-gjliA`Fx8BsM}wsZW9?;Z6rE1l(qrvJJ$r*@CH~yK-BtL z(;$urTX=feG&{u5QcbfHEnqb^3xdCx^nS~ycyC}*iw3BBFLH5%Dr54H!`?Em9P&f*-cDcb0BY6|a_B}9KTfyAe9%}} zam6^Y=+*{4PYX%m7Z8R9a)!yd!2O?rwBo({v_6OCy<^2~eB@KN{RS0Q_Wbq}65-(%Qyk86=dXqb!Z(ZB`Qq2Q8AR{471 zlIqGEHJF3aJwRnZF-MI7Mf1Z5fThcs&aAL!)=GxyU@;^YuA}pi9S$fq!O|8wi>#AA z0h?${0*NZYgSgo(Te3>9b_E-&7&zC!d8Pmu92KGf_6wa^LJCna!~f#wFBu6fSx7dZ z4Rc$wFxn!~bFIMEIp{ZSQ9$@9FlrSnB`Y{;tI#{0RY;AiAn%VX%TbT637M{-=@yvr zg^9?f)IPB8X$b2vBu&he_piGTEKvui0zTk2wpU>x+P9Evxu6Aa;h1}eggI~oFG)|*<$BWBQl%jN!ZH5sk#+#i48)JoGU~FSqWUGy(_vOaoI37w9 z_W~X)ItM@{ljgXPPh?Ly*+ibyiZUv9fvBW<;#Ei; z+E6wR@Iod4-DM#bX|@0ZL{|U;N_5UN1T`L~qnQSLW(1!ZgU_r7KoFNM|C$x&dNzf; z)ZlCskMR;6$*)o+ndS`B6zZGCY~F1X$4IouHVFBP7@Tha$hj02SyX*aE`W>Rz-BdA z#tp`g%WY}F^v}(DyG7QG4?sa}dDwA0fdtl~QbqQ~I?BC(xP%Oan)hL`jWS+@{Z2+) zff_zzjMI{%|1XuMj%Dl#!!lTSQ#}()m!T@Ivjmw@+c;OfI9% z7CALt8fWlqRoa$btwe@`l&F4wV&#$a1+qW}Q>=#5t1DsUumz!*<9xo#-eJX`t+e_nMJ4UpY%ce5&JAgfJw@M+mY5{IuJwN{E7Z}Z|x zfJk@yMBZ;R*sl?`Q9Cg1dcO#qhTNBi`Q&85?U2h0kK{n%kME6cDIhDnSS&Q+M^HY-fy z^9dCN6Gb-3%AM`QN(LdwWx;s~9h>wM={uZspGo$zmdctW+p+1O`rxAL#_xigU|an^ z%*cZq71LV4pmn4uz&`w3(@-!N8Ald_GP_I$c}W2#qe53C^!w%WB z(q;YMv$z_AbvInOI-jMo5_H4Ixu1>o1Q^KX6onTtQBfe=l7bFLiXlW2N6IQt8vrV+r(woxk&{Q3usa_@N({kj()fid= zW>K@#{>?2vN(%iDc8nEd6^xz~T3WuPfo~=_Y3S4csT_p#fNnmU-;&SIcjwdj`T1mi zKI^pk(!wi?cMNZ2EBrauM-V@J35d>xHZq9Q#+wy@(3_Ln){2I|pn&8l5x`ICJFso( ziXDG#?bJVyQoYli<-Z;d#<~U0Q(6n3r-hjyKRz;tzy1`Dbb3-9Jf!bohS%mD)T6)S zk%9vB{mc3uW_gV

AB)H0!lm>V!V8ksSTe=Xhl1K{?qvyumqyrOURYs{AzL?d9%F zVs+pc7UkrXzd&USj64Df{Lx?WC=G;(A*j&^Z;deSOvJUg3|4{^>nuSkBi7qd13nu! zTB;iSf5$i`mD^8P+CQF8U6e=C0 z5XF+Zq}0&=_cn(#PCO!sD{>@TV~(O9a3t*`g&xwYfsmnm!i@VpF`+J`Mk+zqxKzoJ zWdF#+=TkF1L0*j|HH$ef>NR;$wOJ{4 ziVkv_tX33k3HI@j$@cV9FOj7f(50hxBp=8?h^){RzzV*IshMmHMF+m1SUX1AV8|&t zE6@1JmD!bK_*=@lMzci0o||)f%DMS)G|S8Hhn#m!xfyj10k(iZL%lJX z9t2^30D3$cHXk132hYt?^uW9bD8M(sY&5xqb?=xTfu_4r4%QBgl3Rt? zhK($E9%jQGwzhv3wUM%jZkvQV;euQB)$Biz;S-)FCu4kKD#KrE%E?5LYw9RR?9-}E z$=eYL^lpG&*4L(!IPInhT-rOb`~OaJ?J5c{3m?-DIU`($F$}?~?8XVhd0;PtaRv`r z&O!mM8@<_3)i=AO@v|7(QbdX6R<5WmVNjyayd6S7(D&`!01#trGF`?8IAJa;- zb*aeQO@-XU@v6uelbr0)*Fk_3mhu7CX~oNZ>42jqKTqpEu8~tU1h@40X zNl%n7%l3o{XWck0BKJ6gC_4wj8}XobLKh=3w+8MfnAGe=^PN_n^m=V?psb!o+jgV6 zzJ3-_O!7unmdmD>dkAS|05NSAEzRWaA7FmTK8#H>lqocVvg$)|g=(5uqozxkG#|8$RVj{N3 z@BHHR0=R@JHo!ff7pZ)YI0hR_Pd?Om!uI73XsD0qIIbEa{Pr}jnT0gKZe7kS9?_~+Nqf{oh03H8BdsosQa}iizs?%lHjf_3N%SQIw&L2&t zMMTyxngVUAu?hyYK+7!H=!~7hMzC_^7ZEJ%ayloSOEKaF#!uw;N)$$zv!oqI_baWxfyaDbtUs_1Fm11OJc9lG6O*i!s^!-zN+SBlr5D8dgvRnk2!v znu;&2O(hef0IwQ4r!7WluP|lPmpEvd?r2(7-@8tvMG0;kV55DNAVO?x0l#cS|v*$o*#!A&J9Q(qU zq=97L9g}?Np)JvnKg4E4(Qww~-mc_cwBW-WZs{ZUO1vp!XVhT>IZR4KU;Vhi6Sjwk z4NT<)wBSK0YIaJKtK`gSdW787{7*tP6Y}*-^H=nZK}j;TGt()p3XE!)y;=g`#&H1D zBGfAu>`JyMranMGa8yQ+T8O{Ya=Ag5-}mvGJY0J?v&F|V37vdjCBn9LC08O}$S

ZhgZ)2cZ?qypy=HuO=(AVjqDt$zG1sT@Tw8h)qIn~2nxpK zpwP2x0L*z7ekJI1*s?W%eCf`|)uE$9L1$?waw+NkOr9mu80bywz>!42j@pecyyy}b zas_&KxrIwh1Cugr(R*Q#={Ld~V$!f7Kq(0;XE{9l>auE%&lPSohhPQ)nG zusPtu(4oHG%4g=m0>{DL%6D7=pzIk6a&P7P4)bK2z2?__3zU=$;qdBN9`$Pv+s8Fj zG;o0ocJ7V??GYIkS-7cFf=uXoR6_<`jY@<)>pcmma{!bffdJL60acs^pt2HBj!(~{ zN{I&%kFLHBVvG&{$QRUB0pR-{BO-wU^h=kF{1!PzdQ7O zdUaz=oN#&;wHoWAEGi;vxdK|;QE%nqA5zyf#goiK-b_6-VFSbDFdex1mvgj z86mTXBjuNk^r*QX$M>=TJ-*d6X2K|^ZZysa8DFqf6_@!y%}c~9+SPhSRJjsTS3}vy zdAuIyt#eqY;AT zO5~YXH}~-e>$uh=sS;m$pgQ32`V<#}XuIaMGSF^@*z@(^;g!L^llpyIAcI04edDA6GNvT_u_ z3WfQgfAU|qJ6I|eMp2!=I7<9h!^OdxE6gCbvHJ}EIsFyQNank(PCH36j*KBSC>XGk zddzYT2Frh)4Stxwo0qMvA-KY~d*P4lJMm7~f#$cKdDhYy?M}BhGut1`%`Xf$ZQioD z^-R{8w<9tMt%fbd3_gQxB69PK=REOA-?n4N-|?O2KjEAjuR=$* zzaPfDBUI#I)?O}u2458g$yvnV+oWHC!xbW$szwFFAHQF^Ob>F$-ZdsmSu8{(mP}s3CwzdDgDaXty5Ka1^lu;AkZ8Xg(ubQv3V-a7A8hG| zKZ##ABWuU?z_l{i)DM3e-wgZV&*E3Y)IJfv&iBIu@#|ba{CWI3=!Z}GSN2Lz=9f=3 zeq!?Ybo`m~cBUWxBG!W$|7HA&so<~T*G@k?7{6j+`D_1*4_i?Q;cw#4Bm9Ju{H^`O zwK{_c*O8oBVIF`5;H8wB4fcSVl2Hi_Q4pcbfvr5Fjn8-^JF&NXHAVl~D z&do2=pFN{_)1SV(@KCDJK>8DTdJc~U#XS1ccbo0s;~~<)#de$yq(l}23WTlCT|D?M zLd%7*(g`jsGV%~!Br_-Uji44#@B%Q3t;8^Anv!}pxCBVLq^=MJWb~&}5QtgOR->#^ zCw?bMYb1($g~J38BimIqEOM(+Bio<`7S5-t2MgZv@&N5S0ocsoJ2XriwpJh;oCkzm z)p_>ZleJK$9o*kU^18szs=$L|1v+631AtaJDqjU!)s3byiTv|p%?ZAakrauG{=W8l zVJi(H-H^hUyLd`;Is3!1*pN?IOAsE?5Az@B9>AGw4_uNH?uak+3hn3>+?A-5KQq#= zW+b^9XN6YZ17V50qy4^NKuFqqQAMa>>| z*FA{-PV~SCCwu5j^e{K2hwp0ka6#R})7JMu*wADTGZQ_`Pw4@`bR_Z^@_weA@(Y}^ zT*dx4ADo(RiEVDGdH~N&lPux?J#$YG3Nt82`~gjanz*%ncl3OB0E8$K&)DvuLC07=gukMJ;}4iMRB-RycBh)em_Yyx+_q<8EZH!5 z{Mdi#<3$Z*Gm-D)+gJ_DOj|$N&WhOSYgQu4seC)E;cbbtKJ0);Mm2;p(<+ z#O>x7Bl~`oF^;}DV`QZ##|S^E7T-dv0POe!W>GybR<^Fnw>3&w8`~!zY~#T?WO>}Q zhg`t0z9e@nu78NWzy9GFJZy$$47e+ziEor@uKTOn-I z6hBPs3K-+hFe-kWM0Lm6q~b@OYsLuwJ%XZQR@6cxsAAk0nEZGgd$pJm4a${33#gpc zz>EHbGlgeT3((|}A6!X_!F8r?zW95{I8_1IymG_SfzTBA#KdQ(h7SYigkod(>=1m4 z2p=3a)8T`5H4Q$q<*xhs;iKT~5*)4&BEm~U&1hT3ANdxg+?DV)z01u)R3f(%S=Ju?xt5Mn}S7WOAG|fdvY^Y13^l*vjWwFSv^=cRF=D?OQEoQ z3X`=V?SwfnL>m+?98Fixv2Oa$CU~FiCmU_LLW=N5x)|M&hGo%}BS-dHuH>X0ecRqN z$8Kj}SZ%op){U{xZ#JdY$9+G|7R~!(EiN`&T=W))Ws9dXxbv#Po!@Ngd~fQ!vZ-e^ z2luID{Uk4}+PS3J&L!T?g=IT?o9*D`%-6THr)ue$&6b|&E$!K;rT+SsOer)lJ-6A? zbG@a@Hfrhd>svD4tF=T9uiB!1&|A8uZ0XYG?4Gv1ndengyQ10572eGAPPdsWt7d+r z+02i4Ggp?)Ov3v7^#i)9YUu^dmR{g3T~)U9L(I!Cykok}LVx9zYd0LdbM5_dVdSq| zd03dwf62>lyv_X;=H|~`O;t=Un-P0XNEK@%5_LO1i2lDRn>})Pl&+L*Yv-8HCu7M)DMNlYYNDt5S=OA*V-93YW)B@^(~fxU)Hw zoj#HsWs8qWr;4hbbDHg(J1=Vkpcpk%NASZJ7A?3N?su=$;BfpQ*&#k8)yh;5 z{5Ok=JdsyuX<2ZJc3Kl+D_MwQOY)Y(VLg8i!WqF4g{-jc@G;^)^{EI{+2)(ptg&{i z%gimDc^%vpM+v@Sj6+fk@6ss4@h>&TDo@iXMoOTSxvI@OwN!f5Y#qBtr4`|A8QVu6ws=&Am9IFwHfSXQCd-08}|)81$y>6ilB9T-a>Gd0#3 z6>O{}0=Pkrm1HM1jSPiJ6_}bWMT4MT9WZz#b^4fvry)2qx3;Z!N){+iy1Z;n`77CU zntT_+CK@|}fLtlGwofg~o{P8eIb1gQ9~3&3exGk=VzE-t8+_zQG1Je&=j>Q3E*wQ5&A}*1 zc6s0cRURh!1ZKb?2F2MM0OEZSi2dsC2p#PaEVyu_^_hU z(%ZWw9reAqFu)5I^DGPBb$$q`81a>is%lZEM&`oBq9l}L7qIL+Vyd{8@ewK0Fl2gp zs90pZ6r5BRx!4OK@9#?3y%vGZDfCOOmcSkcoVZe8E4Fyft8l<2%)JCCqnIkX9)%P^ zIU(4Mo!Hq2<|&~G??3ueKXJ9paXh(NCr3B~hRaE(^Sj~G$&b;RdetC^vSDD{UC{zJ zgL06QjK!X*2HGriMPrd#m1BNt>RRi0?TyY$%(y+ao# zzs9N)0L#@~HWzMOIyXO_avvlBf1%r^uSN;u*I_ITe5|FQBcJ$O3>UEVae>ed;+Zf( z@JYH|+cP(d2+`Oz+?~S5F)oGC>!GwDxQvz*BLWvK@lkMdBp*AL&(z0x++fMah1^}q z#~yCLdqh?f@ULYB6=HWv1>HZT?RsCabyiHb%eMTHk>BRDC<2@m*DFM>0b1kd41SN;%iBTPnF9! zwF0YmE>%3&6P=BRu!)ihHLAt42LJ>_;y3~yjT3n4zbdFj6-wP#89d%FpLYott1g=0 zEI<~SGJE|TBqBZvj(_o?w8};RJY=;zSzXe|NGU0?Bwj3R?+V#1@OX>+F}X zvZ_j*gkWN1BMCp3y%?GvMFMyjxd7ob>5JKQ9az+hM!ie90@gOCp0eF}{psZDeU~1v z(>ID$5SnOw;l73NnR%N34X%%o?UwK%-WWN(#Hs9s4zu%@?V<9-0}=AXkAulb124|w zt}E+X*$Z_zJ2!B(1+E_CnPbID94N5y-oS>%l_-k}Y!YPRPl3&H3G-o(9BhEm!KQab ze*$cC71&e@^P*Mu zQ6fZl`POYh#^_=FK(M>a0mQd!gf)D=P!Q-ewGCsqAsZAIM>K)VSd8d859W%&0nuUL(kUVewlC$8&a}@K z3(q3;YHH#x%LdzdHmsCtEX-9E>bwAOJY#>c3GQ@2tXA*qvnx=lVzY5zp@1*6-FXD% zEHT(Gc=2|hr{Prfjh)cjl}2V@OE6<2kykL9&hSr#H%w0a1g}jbji^q$5-ixM+%>6X z!z7?NVjonOaYXF%A2ui?e3M=4kyEqk?nme@dDQN(+)s7~64%}N4~Kze(`mo}LPu6; z;{sRNXFZTML4&~H1x0uGKQ!Ty5^`I+aZUiNf-RIi67fdkO3I@q_V}WaTMP2}e!#sm`kh43|coVnLRq zK56wPr30WH>=YW7lp)A#L|tSnL*166F3Li*8I)sA!^d?Qjc~*U>gU3&DXe*RGbo>n zg<0Xs$4eYp%Y^B4%b|Z$=R{U1xt*-#yvg;w5PVX7JI9VGh%KoCPDGvjRtYXs)D6ZQW!D(*pUVREl29%z-)yrBGozYub1LsKBl% zRLPG?v!BOb10RV4qDt&TCTc{vQ&J^bic70%qDpaTb*jnCK>%cjBPQQ=+U>T%EG6aY zou|B^arIC@M8k0q+UlL}t7l7Kjul(IOYEeNOF4j9t7p$IG`&+Ra8=de3m7lti*)~o z4wyAThIiY1q_et|xpprldjZSU|3b>qgzz7wtCir-Cxs8KDvU$WnC~c#zTJ^MEr2ks z@E>sUO70KbxuhEjGEeNJ?BZnTkqer4tPEN6`Nh`C73$R@QI4H91D*8iuq?tY)*5L~ zPHx(E!I~$CawX7SSk169<#UjuLhvh;tY3Bzpcc^AA)kJw*T4$9ev-!#@iDK=IeRB9&eN5|RQXD~D^d*8Ql&4>kX2^1; z0*0GBx*et?b%U>Wq^2Q)4fKmS-faIEK;u>_)kVyseE;8Ga>Hn_dlH`{1k?RjJEjOQ5sA@Xq2S#Ut6x;EmbYJR|`D zj#{;r>b~@*YWeXDN~Fz#6<;5u4RrYF2UKC-NVFFF*^jY5`M8<~h{qgjfQ0B8#ncEd z*~_@>56VsKj3effY&lhms_3lIXEBB%@{Y%x%ghlHSD4)J zu^n2a1p^4b9(E!#Qj8{(#_nQD0Z$)@H z=V^{)*GJScq%w<9wsPbLZF$x{bMr48X703=JbR>5!}5KcFxJ#QQJOEHTsA?>0!%9- zGReNMlrj|X^7cRYRSXLI4*O6U($6j!ooYK zY*qG@)dQ%_f*id??dDoeA58f{YP*7QQW{aR{0s^wFiIj@XAa7^ec}%^3h*Bas@I+mlLqY40V{9 z_Kw^({E~_AcB2NPtdlKQo^hL>h(oC-rs$Oh0y>pIK|?G+pKkx~3|{dJaQ4Xm=~FDG@}ciR>vtpht#E}`Cu!*;L3Xy(Lq zE9_KQrNoD;P{XfXk)~Q&P@(h1ntjg3DXH}i>S0@Mp6A@}K zEZL0WFp-eQ85Z2BvI(J0gIE-9GAv^ulY9zlH_4@zv+(AV#Ijl6O@5U<27V=cZ}O`K zlM>De#l*#%ysCeG6-T1i!!L`fuxry$4BsaYgb^p@*Bd^hSCTTa{%i7>Ca!=LA*xtL z4|br`3KJ##hOG=4bwn-_-GpYD-cZ#PI$%i@O>b1NuqAxc!#4jgV`^I$Q|QJA zs06dKByd#mRNn4;u)J?Fe=6^r3PXg)3VZsuX8Kf>f-e{?=!TzAgzlMg4W|Wnw7q$4 zt;K~yBI8}c;t?YiaRd9CPeHPLQhNE~U?cX4_nMCGY(tyC7tk z!Ax1Bzym~BnER*)h{sClz9V!te7GWy_(|k?Li^+FyjBznT;)}?w8#A$|0OL45jld# zMgH*?!*Sprr%iE%amf8j-?f_FiRuERy~WlJB$YtEe?y{+hlpEWV9EQA}ALjR@yiQ%NV=nS>v0X;PJ8isg zkIv&%(_fM@&NuqNA0O_s>EYr>eCMWgLFw~W>D{Hrxgk!9Y1pU`EBF#6e{ROC5^cOK zOEjJ4KfCB;zUOAEZi0UDk(Y}6?FbL+b6Do@Vr&VFRs(3EzPsqL^0+Lbh+AF%tA zH$44CG{eW$`(O*UMX^f|iU$l&tQ4>0du#Q?4bq~8D+c*+ThzsE z`4WMfZ{KFY5yD@Q>%5`HERcPcD$Si5^Z`VbeHDOF!@-!jZwoj4wM5QVT&%WKJ; zI4KeuA(@oJe@Qgl!BvUBjK!q%0}OzfMei}kT_n3=syHgkPqb9HpIQ?--2^;X8zgAY@gFekRMUT7C8K!W9tH9O-qRbaBFe6+yG9C0^jU?q5b zKw(#dK1g*}Zyh0kAE&jbzY%0x>2D)8oB7)fVXY1^y>@N_Wt0x_ zmEnSMWeH!|$gXh*wz7eKP6hHqMB;p}JVMgZPSKT&Ei%qoq>-(TZMsv9jmB5^X{##~ zJ*|Q!wC~0dA44W;H*pIVetdvosEh|pfbsw-@4r&_3!#{$wzsvZG(*`>>;a5@e6t(D zX!&M`V(7g#yHO5>JCw9z)bY)pl+!V~RK51iE(g@K&7Ot0*>PiFx71?#W@oYGjk0at z9-FNNI?EJn+niCvZ9Xw)LCowcle_%sAcz(k!HY?W(T|Kiu>oUVR#=D`X8wv|oI!8CI+5ci%`%j3`%AM3ll97e3fTPSm#QAEM$=(~D zEW{Z8iX;MAcqI?IWfB@jLX}G`hkfS94S1y9qOvE`y;%(==g26JYLqfb*D`R{-z<>aDKK$#+)DVPcaA+;1atxwf7lAc|1N;` zCm#NuJv_@xj+({&jf#g)|D4FBU}lV6ZKQS6aR3-RTVuCDWX+|B3!nLw-7XH2(R`Rqg znUISnN9LCCMkPY=Y{rR;GZ?e1EBzvbxf;<2zBE2VL+J2dY$EgF9(|TFz96e{l}9X! zR{wv)kf%jZS7z_jxldb;_H#xZkdsLf9!3Fk0ck+u{4Ba(KN4p(+W- zO~Z%n;av3)dY^{hw}n}u zuPl7If_mK0^%j2Tt?Qu9a54)rXXO;oJf#8cNvFYKvH_F&iI>x*D{1(uO|>Ly3J(FW ze58`<$L4niWL+J~M8j~`63rlSCh|3FWyT@nkACtETI;wj{^-^AC@yM$CK>*W%q~g) z>KbsIC(NSe1y;g=UacbH`O{R%Ay4J-2xoKV>#HZ%1c~A5@HJ-5VTy!0G1}oz>-X@r z-r3)eFH8Mja8QdOoFlFH$ml3ejZFqy2eA4O3oPI1S8w>)VczIz5d@vRSSp6^W zn~%lUjkBGoUaSb3q5enNh=76fMW*My7aY9v70HJ`V`GLe)oW6;`xj=)EnBOyX^Dwo zo}vSyk+aBjazX1fe7%FWuPMIQzyH+u9OK@Z3Tt+@++{Y$>F{Qb+AkweT5}|Fn}uQt zlCSnbC{$gW5Xp{?+Jq+6I$ze|F(27erXhh7EY@3-Qy?OSpS!*MGX3x~u<>ru7V_Ka z`E_&r)o+xzW30rjWr?YR>Ju4AICt4*!$n>`U|7A&tO{~x^WiZ-v z+7_{4=^TKU!5l!4oCxt==G9p%}l5@gpCTHq_XjaTY zL;b&Z+wk?;O@v7(HxSE6h@U7$a&ba-W&MI31(yG-4}L_&M55YZ8=;&$)?g$Cu^>*m zU}B87XtNMU|8F6;y`ItRbZ-o^lNu6dH!GO=?24Y@mkxXM)T-vj6fx$>93sHD^1|sD zz)T2^uAZSIzG&16WDhqTr^E2kB)m&sD$b|#tV~_WR*8Bj$e6=XH(3%eQe)xvE0{@; z!~j4mty&At4LnRG<}8PSMRW;NwE;7K-;FkCTR%=OOzcM9#oefr4g9Khqi$O(d1ivJ zO0+lgd)NjYRTG)(0N2>FaF5OJL)fIp=I4@w)B3igh6(Qbz&MFsk9qDYp3ZlRziAFLo60KNTV*$1`Wu0-^$jjO8 z^KufV^wwknqv!rBSyz&l#M02}*{xyi7r{g+1+ZBs^Ca9ZUb&Lrz~;km0B0r{=Ak?YNewwJu@)0WMF_CBdG0Cuy&7EP%ag#^LEvt@TaXXjD$DgV|C%- z4P_t7SprWr$%B*QEQYHE64RnZL=u;hw1mI)u7@|q;j1^z!AJ2@JsrNx8Ui$hjE8(qh8^Un*cohj{wpSM@ABLUzB{TRO1 zM<`vOvRWnmw<=URGW+AbN}tU3gf&hNXchzuB>a$6>kD3hg?hTy8=erVc*G(%0Go1Y z>cC)i_)B^@v4x(Hmy`bFeNi*|ezWcvp4b*ziE3L|&OwuWguH2KCoALrg<;6t z!s#p=@2#F=Dj*pBv9|~?PYe(0vtq5_)w%S)J3rP7$E?J+Q|}lHd87Th$4bX&0d@rg z@tyEDS=x5fo_4Y9+$;sck6q-cy&YO6h1%H*dzppph(o;is&ZLzakgX!)kHSxigkC9 z5d8C+`1wNnY_0Mi0%n&rq5%mOIGiOlSoU*^V3vTOP<5n~ZrL8+RS8){cM&T{-I$Uj zG&}u=PNeRn40Ejz@+FHX9ZukWlQxZTiyRu}ZbmGmT^7jY2Q)j=dM&1wp~vd#sH$!) zBTWJg8zUVt9vwz(`>Nr2muC&x(-j$HZr$BeF-{gz6rJ=9 z8B&|zzZ)2S8lgxKv}`sdMHfoq$dh-7f1o99&Leg{h$WG*oP?!P!0q=htN9$pn2u8v z78S-h(AwJM*pv{BkNqcz3typx{?&~3!}o~dCH*Cs(A!0Y{w@i(+}0pN_hxk$r(mVI zc&Rp}tn-0$-VFWbchQb>e}3KlZu=z``jyHcDioS@YrHesM6vaG*hr!{Km!GT94HVw zb4OVWL*rd)-#IthFk+g?QzH{F8IP#1bnc;Z-?8ujjb(}V|8U9?{?7kA2`J4P8AoXB zTnw7WQo^-;q8Tk~vM-|)wXT}_f<=L7jk`2NH}o&8LF^)DaU^y@ipXKn9WMmZueFA6 z5!Hv#t9rhtRwi5F9%TaPFvg{!FnediR1{1JNc@J|F_>fsr_88V|ym}AMx@46L?-p%D@ifO!-lW3VXFQtyGXPL67V#aV;u3_Y{pC8E-_}oY1}vVaj*LAZlcsbk11l->A&0!ph+$$v_aJF4r^EH&LiV=E8wCOT%Q}dMoKGaEAz`x*S_uL1?;;NpaLX-C+Pe~Ea2=-^43Vvi4 zr;iW!{(?GwJY^q`T4z5#qrbyC?$$$ewd8y=cX3*eOF}}8uhHNGOt_{~=i0tnaSlS4 zVn`(IL`R*amdrN20{lpXqU`T#`TPGp%X5se&Fzqta|LN)z#4+HwAW<-%T^u!vMnu| zLeG!%iIzi6`R`iolJ?3Y>C0F0@3*fQaYx)IfE_kN9E%&>%xso!3X|Q$MmIBC`EJb2 zh8$5zA;OM`Atyx=!zLlxV3aq4z3E>|q0JJ0#|qsK>NN_mkIMKD8~lNC4vK|CeAXUo zzR+p+qJ0}rtDPi<9O2dKn;+#p0#4GuXl%*S5~l73G;R&r+$pI?LIUPr?DN0rRewIM zc?uF%tPA)giim1rhKwj?BvkB{2Zx-2iBvGc+i#Cevvk_HtRKilQj@Zma&A^Ux`x6% zk50!U%Ou85t8#m@ z%KyyF`BTp&QXApGhYbf1bF?h>qYm~iwOjTy=pF?`7Hkkjc@wYAZT}ln-1g43qTBv` z!WHM@q9DWnrr$E6%~!@bu!y#a0r;9wnVtDxRYp012oS(5<}JyPrTyP5nr=|k^zgFi zji&XyF|FeYn$6lmz~{vpA7!p@{fCnp7GtQJYR*;TD-BtFq@H$ziwfz-fmqsc2)=({!<`@giUBH? z{N>;L@~1G=pscagQw0? ziy)pBmC5XxW0gzZnMn#t!_Uec9Qu`ElO5{TRY*=nsfx7!v2LCnY1PMx?rSxyj1E-u zZUT6dkzsX1*Zib0z!iWrUQ+JFG#Ywte!tdo)OsGEzC%0);tA-W0AyB2nW3zuB?8n{ z*_A}8!v*qKL+>6`J4LIkC4|hw4AB$51Jm%3aWMVvX`EFZ&Mng({hpgYpf=2`*OJ7R zX)Fk8I>sGeB1V!3@cASe+U=!_j(G3?OWxZ+TUwoUp6|!`y62pG@2O7;s-WI;QpRFT z39e}&hB4N@vqFNXAwQpQd5a-6{s8v00HQ6l`Lo zbZT?Jtb1nGxjKJT8MzDTHt1aJ%z`@4skTPU9#clNCqm^%lv|eLffDG{gUWfIDZB55 zVjlXO)8&H*OCJ=~#5sh4+$T{xiv&g$jX4o*_WDSea;^UPN164P|MzC_;p$tyVu zIFsrM6L`S=rIIz-*fS?b@vT#RlWV*tRG!RENI z0gO41YpyR360jd3{!oPC7qu^9vyXQwRde!ln(}1yL(-7nA`nh%yqm6C1}!Do30mQ~A92Kk(YTd}38+>87>%wDh-)vbu2y`J z3C7HX+3t`LBuTJTZx-nL+{ zvJwGdLF?Ffbs8X6aZWt}<53M8LLwF5kUt=j=L6i}s_dVT#!6uf=SqszR=h4-7&Y!U zkXlhK&#$snly6-Ctbf91O(0-HSyHZ&-6P4X)yDr2*KzSe4x9QD@l4Lt)=6KM-OE2} zg3n*F6I8<~C`|2~bXN-jY*i@v21O2CV84FPsONnBf)`@}z#PO0$LgFwtDc_Gi|9j9 zH4ew!Z8&0D{zgEDe)tVOEr1CxJ^FW}1|Ost>ld+CUec{PsvlILZbFk5OpLG6ADbj73<+iQ=ooBt!mEg zgN&Y&UsV#2V?BFJtNg_B2zUsm3!CASIDt3AAjKP?UhxJqLz&lF8ls#FWOch{3dM-m zJfQZ}flBcq0J?I#w%8SQOWE%<@D-s3VOjQw44eXXYoGn)2Y&Gr?|R~|{=ej1Ci<*B z|IYXQ$`Akful+y2bj!QW++ZsY`dFiivfvND?`Pis;<=B#=c%&b_dWHqzwmP({?uRm z`{Z3`){-{?yIHJ@^_{EsEQ&Lgb!7kk1V&6Atp7`Yi298p80HIU`LV{{;0yi*Z7M^ zf_e?1)P}6Z=E^-+TV?;us}*f@04xH^bXT+kPBeTlqaSHdD(Dksb02lZ8rW#b*(@cfp@ec@E_i^zTTMYwLPSgFQBMQEC3dvThJjN5b)2X+!AcSI&* z9rh1cBB{r`IZ|eLtK_v>Rlr*NKC@wdhF}jp)Q)>F5YohKajTgZtK|w&3;|BHwSv~? z0$Y7sR=rXE-^q?JI^@umU(G;F8$8WH`RfrWY!apk<6}u@gPruk$uW0)=(*ro_&@n0 z)vFD9Jb$L~{Is=qyd|A9=F94r2`jP;L?&a*aL(`^)3GFfgZWPyP~wpd&d$Eyzi+e0 z;(w&cm+tl-H@f<45;T6b;toX~4-^00-ZHi-5XWQ2cr#cB6>6B$V8ngBo$B^~cBr7WA zh1pf*g>x@#wQ?qgzAKv!J5>)ncN%t{3~#l@I^<^wfe)+cRO=;uGIlcgl!$8zTWRXO zpPplYHIug?XIjoIltiyNf3u^3+VRqOL?h7!lIx*Gt*87b#MRyu)CHz`h^u^8jNL+9 zbCx!$H1D9LBjCDiBv&LbOW%8JyZnaqY^fv{>z zQ;u{%24huq)W0e2PIC>#z4T_z&9x1n{B|D29i`mnnr$ErqUw%*sqNkTuZ5KW)zp?n zUyh(5=Gx+{U;r>P{&<*{Wg_TU1w>7vHtiyHMpXJ&DS}v}KQ;QNw;#}bRlm+c9BI}t=NpTVZnS6n zw()Qv#)iA)b4C93*eZMCWlA9!GM2#XF=bFB2RxsvapGGeyNfK;#mpl6?o@L2Dx`p*D*I*uC;>cKn2d%+^m#9wPrNqLYlAbLa#^HE1e`uxh8Z z9LrF_a?JJz)o;r&?ra?$4Zrc_ScjkRk);?$Q!d3`ATGsLg0-gTdBMIf#a7V1DS(P^ z%F?fvVPj ze8zqmSvY4;&w090cX(VcWfX6Nv*bH#gELrmMDBWifSvA*NH=>B@LB+7lzPsqvlo zQf6k|hIWdAi!mr-A5j;RmV!&dzAXx_RY_DyqZY+$OxBtju9a+SxHtkBg-{)p7A)!& ziK(<0?gb5E{KEH|R9)@zWs1qTcU#DfGU0+UCk-mH|HBu6>% z3dA^2hSMCQsUga7suP5ET^gY+@krPuu?&0thPgJPl=$-M{=Tdu>#c27V>t9X5)Clu zV3|eE10SM*5r0jRP&>kV_Z2JR~k% z(7?9R1;Sxs;T5vg@NA-`?;9k(dQQ5q-dl7Z-!z{MHM1p2_FTFE=6M;U8j0YG$v+M= zWssk$^!>;zoO+AfyjMr1i4WcaP6&qe*U=l2Sk{*!IU+5SFA|Q9a%`)Z=vig)4`70- zD)Ke{%TDy+8Jnb=QI3+vC)L0-k8zc3pp21q7OjPMQq?gy zH+*f|06R(saK0n7-;kYe37lma)L#Bw|{XF4e2?N4gh3?MK-;ckO3hB zW44HJX5E1aUC#a%e0PeR<&&ji_y_H^x;))R0cxlm=|7n@{W|qhvVFfXa4zWgAu8p| z`TLEFb8Ej3so&a?Gv9AHkyx!bu-!qUWyMA=54rjv92f8$d7(7{z;Twj6a)BWxiTz@ zw6VEKbE&X2E2&0fHH+X>GX42eet!K6FqS;kFpzw_@%&k<<5I9mJ(`?ll<2!2YV?XS8{NxGCIn!Ob&c4^T+fZn-*pRL@EdEj{RuvIE~A9~<@KMZGF}Ja&`EbM@m#^_a`R z^Qg+feZoiW!8s7CZ+OVx2|7o%3goPr;2D)ke$jmne&3GZ;gY;X=A!%dW84|4PqIQnfxhy|$|rM}?0#~cs5<^p zFZIe1TDDYpzsDF>^1g})UtmPu4dZ7 z75kUJ9Y=VhheU0thpf)lL-f1YEBa+)hR-VIna|X!o|4tONuTU$TDEQ-i;&UHMnsgT z+j09Ko@126{N#z*683j>qF7?o2BD%mT@sY=Q4OY?nuOU@dX}M=wy%Rn3JBmbU3Nn# zXi>DH{#pY((kTNG08_hn8n}jX?hn5c#tRHSt?5=YFMYIa;04M=b~M-YQetAK+3~Pj zyL)9$t+LLbyMn*h%@SP;!jYgvld|s)4QQ;bB2>yX@KnPCUO=6TK;H}-Y@b4$`>`%8 z^jy1VCfN;$mhy3vd!jT7!4-Ql`*&I^9&!n8)@Jd1S1%OjV+(a50(ElNtV?G1v6%y6Hf5MF<<{UF{?`$Ps{Z9rn|4QxDa&-&LGfgbLI&K#qLa zH&wrQ5MnF|F?fbev=Bpm_goP%mNe-D5d^s-Vk~Lm2kPwrl{X>=NZ3LQd#8X9qvwb* zr1Gj`bb|9KAig$flB)ArC$bz{h`h~DfrF7K@^%P@WJC_x1l5ZFTz&uo%F(5OJ(YZED;2HkOQSR-XGn7E*zM$m{w z&(sP{CToMu%sPjX7)xWzj1U%2`cFt|qQ~sB)a7#6{nR^?a9|w7-Gc45Zv>D(?ytM_`KZJ*DuG?_1IKbIWqRP0+w%F?lZ5Q>z6serWad>hzZIY>vG8= z*(iWpRU>DymR+>*_%OH$9rU+7N^~JSnyib?=q{AS(5ZQ&n)R6sHcW>unzU{TwjIby zC)prT8UQ>vov;LL684W2>TUSfOLT+?scPZ%%ev%MSPilb{0B$?Kz11Ns&?b#u&TM_ z$LZn`Kxwh$Rf;y^lUNuXmxgRVG^5CPl0hLx_v&>=g9?_qWe-{~xH1H=mOQe=>85HBT-8uO{7(fBqmjao9ZZL&$&MjDM}pOA16HS7*r z4(PsV(kS!s4uBtJOvS1oOadglbR{nm$lC&{z-}xFP|&==Tj*>et)H)3=qPeX$j;&h z&f_pp1OjIcfu3p&?g3h`Yf?SaERP@u7_u=yt!ktpA3BoFHP4{R<^jJqpBJOpylr%} zx^2$rM{@@0Cx~BRhHyqxBKE=RW~f63K?^ri5M?SRpki%|&-Eu8Y< zJRyve$zVxuhNoA{6T$lc=-OxA7NavzHjE|nAyykfDX?Lmkz(Z3ip9uHY_%m`%Qjb@ zBL8dd==jErxe*E=v`Er7&aE2~B!Pi-mDFrsgbJ>nfC7A$%q)zdcjT^QX33u{#UfnE zCx(6m76=Qug9t{0C`|?=K5C^JXVq%&mHb^K@s(i+2a25Vb?p{H;0q=?gC^>Gr zlM-Jj|gaK_FNI{>$E@0CprDjme)vt<3ZZR$xeL_c&m3L6I31rD&i|1ZtO&N{@`Po?c^a(tMKXMIH$ShPfS~aIgv4L@2ov z3aVp+Q0!z!BNSi`6-u2n&`aNCL}Px0D0C^33vEUPDTiE`!UYNv-OGAe^B2{`L=-~sEJt`g)uydo;KmsL!>;6 z`2w+WNv09w5hYIIG@97)G7?`SPTErgCo%d6Cre#<3r_OkC2`V>2{leKs;xN5h)rUR z%P+4HCt)rXPM*)+;Y_@;b|BXp?{?O>jlIK=5I3Ii0Pf-lYW5CLhKVQ*9bl;$I>b{o z6Y}7fAtM}V{xbtkf@J>R5tmM+_$>B5B{8-2l!=zhoMVNYse?ErgM%y39?csQbAiL9 z5@Zx6z@N3($ks7XhZ(Wr*1@WH?5^sy?jjGVxIb6LLA8uX80aEa1X8mFiK|BJhzWXp zOk%nGEtw%2L4t0syOz#^4Iaqxi|%@`n6@B|t#MIt@^T>(mGLxB5UaE!g4= zv(z$>orqb{gfeJw!YYg0F;k50m{*!@coXHJmG`BU_gR0q*g(>Z>0}#LV$7atRd{A+0hEjObX-2nj|)kOU~IiT6)FCZR62Cs*8P6 z4}~WT1+j+~5qsFcE3pTFB3{b|C!En#o8V)@b|yA1FA9PW`HH@=ahs*Sa7V~y&C?R| z6&n=8R=aAD>U5!q+V`G{c^MnG63Pm^VBC?gp1jArC*QlwC13&^_(-=o43#Me zr_6t!F>$CU7MMcc1O-5a8j#Y4MuX*$DFiD`$Iuz82nChENUiqKSc0gHB1X@Q$1%fT z!YOAc(#~;8T1Iqe(6Uax7wKYb5W>c)><|!**OrR`I(l|gO^O1MW@t_%X=hcFhM8A} zK=Dj&-DC>70Gy+LERT~dIxyqYcfFfkm_tFFY$ZL2N5NC=uydV%n>qG_wE)@~T26x1PIR>MxThU~W#n&9 z4f03D(L!2+L*%bDmsxZ2uVLp)aQ(pzw*~@)jnX}$5l|n>0=U6LE?lq$#Mc9GbVSQ=w$AtjAq zJQF$TEVh?G%l70-i;7E(>JXkLpO)f*@6u#mthBozt$Q!oUb)VoVM23jC8fu2=g>p)=qB$JE#~i z8o9@^y3iDt6%v62JT0d)DgtWJ@wi&C9{7U9E*Y9R-OqN^TEb{_Q>J?Z z1PJy9k~a?SfgN-)Lt_P=`)gN9i!n9GPg@@&XxLGK`NgirpR$8S{>b zO2Rv+7zKMG{3J!#N+S%N{7Vg^@PV*!HF^=R4DTw`OxaPc%el-(K-c3$Qq!5rnp)WR9VzQ;Q0cU3@9Qgncmdik^`Nejp08=#6Sv~AJA*jdf-)A1}cuyn;=CNz7Yr}MD1%uFu9JjiP9t!BAG-tAmSWA zRUQ|2AY70_-+gU2LLN614nSC0k_>C2B?%~33P-n;IBW(|Z8EIbksFleRChBcNe{z_ zM44z#;t7dE(T3s#(VSEcgKlcj3O*(u3P(L=`>?zG062zjOo~&#R8Kt)v*y$b9T(#k zF#wTLaCC`q5bH{E;aiJbA`ehZkSIPo8cv}$!xpi)K+_!h2F4hEs*4z-O*2%yS9#X4 z3)DqsCfU)?k4+zEcSqD3xI;iv%FT$G z5hXL9q;2J9^10khkyw@FW|(nl6fNauaz=S1spJvTqTH;5YMOGh%PfKqg=|D7H`_8I zM`J^77SLyy=%U;#*~aJ^N(Bd;JZ*VppkQ$+H(Tt7h)I2Qxfvn6lA8@G=M!7B-uPD8 zCKhiaH!FGv_D7j0H}eD*<2^Ix6XoU-9AafrZYCdCT*J;j?AAi94^G%NtpT2AT0{YuCUb#-Gw41a*Ebg7nG|3 zYv_)FE+~U>m7&Ji#}@9`UuV_O-!1(SjEfnml_5N%+zCMZk!hhX7Vm7tu*h;J~!PNWbLi>XL7rb$LNRv$`1 zS^P^|eLeUn#X3VEFxHvetuuLPodM)l_yDKsnso+dS-GM*-aAu#;mer*xh+WRIYNoa zm+%XTN|u3opwfN(#4f3WptKgYi{U~t&g5>K0f;irAjRcPff-W+O}N0U4q6bH3(Ph% zw$2Izt@T!W8oQbm35iBhy~+3BeR122b+CleXmpC`c4ry%aUukX60Jcx9H4nlc#vEv zQ>p!Ae$(+S>%3-1i!1`To%y8tTH^C8U1z6K6xJ>3 zmd@%To^q+A(Zc0uHv#4nwlzb{IdU5~7H?@vHjYV#&OuF)2vYXy_4{SRN9xQs%wVL| zn*l%rCc27skQINM`B266L3c@)E?_W0vT+jPQp*akCBu&@6ZKO?I}<4`PzV~Tl3uy) zA@7w*?GxfU)Dz)Uvpu-5WMjM5*;A|mFs>Y;laeNsyz;JH>N5gF?YFc$R zF9Fw?f3j7MaVKjm74&fOV^+27ikh9eYD1)C3K*rN}IHA{o844u+=(ar8m}(HNkD@Vr0to%kgIoReLegAEudsPnPpT8?)rM{U|P zG7Q^GTB9Zgf+0+PTq8%&s{TQ1H~d;Md9R+3yOjE) zdX~-*Gk`iViiNR+M=ETR=&$1U z>AMTwvjhmzoB!27oEGUn6&x*JdXFES^i7s!Ir%yUe!|v5S#X?9gj>GB=O#=}9wo3P z?gy+pb&IO9a^)G#?@Ku|WAZ+AK~0CRo&2PYCM`TD9Q%O{mR^wry2>sgs)#7$)@oyK z(cR>`d-J0<5|DTDbE=SW@C;^9JbW#-99vqz8{`<3C}8ZMkQO*;9gxiEI`EOkwf4Ht zHm8FWm(rmhOq@VS+3A^n0H!Q6S*%In4!%4*lTlShkMxHkI!!GaRm9~obyYRO&Z9zR zc~rjxpTluAi1i3>@jU0udF1gbttbQu8LFPOSCs}h;RWUdHZPr%8UdCr$O)V8JcFd7 zWhLKGkpJeb?>vJnWIwk01`-B0O|v}ag~71tV?kbaT3cO;3EHMYgj}Wue8Qi?EaX6Y zmTSyLsZ6cLOsizdM}LzwM1dlCTU<#rXJ!?%De%>WlB$K1yb@BGIES|;(?u@Yk=KF` zK)T4H)K+9=DPcxN$Fj13mYxeKke3C3`vnyc+!-f7h!O@c3^;Us1DS0e_<)GcN{93L ziuncLZ3;yI>r69)gy0#W;RI1Y2o>ag{5nDmj2F2SNrb&lAA$wmD{Q@_NJdxMRKWgA zCN$&nUL$bB|9Lw=#%^Fr#SQ5{t&F2@w%w&7_`AjFjpv`ylZ;luo`|hfg6vls@?**M z`;oD(NT2EQlz!cBohfgvU_6q-9!aJ=K~7|GlGb_x z|0r)%ND(o7U-Kho@1Rj(N;?<4gEb`?{bmaF13zk0la7vh0*M4;*qX6LYLIAZmGDW_ zEW}!nC_c6&rzTgJhp>_&^2y>Gz8h@Gk4Cg{EYSX9Kc;3YEeZ=PyYVz3IZ|_+VR)0c zBb^<6PwKM@z4$T{2P6K)pe<}j0GSFsAX)euwJIJ76EKy|bS1I{P4PYYaFvaITEhYi z4aX|fuxgc1K@Ij`)ohGxQj*F4t%D2v;19$PTlc}iU$?1Vc$|Ulm3{0<(1HP#M}!oH z6wcC0`9a*YI(cI!xx@FV6q^O_)b%%`5B)uzTrv6A>b#9aG9Zu zL<*bQKkid2emn)XTLkV5;=))mV+vd}sV}#c#OX5uYti7z14T;_DB|U|Bn+fwO+G46 zQI|wdkAUF`lK}Ezc~^*D(6bBh(U_v7H}sNPa=XFj-WqeTvr@Uf0Rc2;Lv|Gt2fSa~ z9jWnyU5+$~BWA~G9))izlY(zTwmMLtsxj2buLzs6qy9&|EnKeQ=@9+q=T*jy)HXE6 z5s9c6TP`BO)NSFC*y5xe*cpSe7h9{+UD5)N3QHN{bO3+ixTt*$@$oakx5dwHHY+H% zI1xJv2=6TIv_yB?j8VC|qR6P~#6plTuWPKTf*paMTI7-|l6U_x)HCZ}lQ0voHf_!L zK}#BFEdeY+%b|BYzdY+)jRhh?XDg;_x0cX;*u+8EV-<+|eUO_}g6Bw8o*37yX7x`+`M{vM-tf*ewjYvjveuBMrdeFDe6;i~iX=c9)xmF{)`58H zw-;h2BIT*4zkC}YG|kArYSp55>bv)o`VrX7w@YW;mFu69(Z9)SE>WxEWmo){0s2ie z{BlsBD)u!LhCtoWmfg76hZc0ou*^2}wR6sl7<$s8$%-MFoO5li(DJL`IwAzuCARh$ zk6sWR!oIiT;_IF;eCiy-&j(%&j)ndSf85ND9Y2mikLjl`xMB2=*#iNG{nz1Q^pGJ{ zN2_Xxbgf+4zGN%EFXozOU2~0$$ryA8HO!rW$Tz^i6o;)FU5I3{;l{%_QnJx(JlX*2 z2N1^Q#7@mTWyV}+D9o+w0@QQ1Z2z{ta=hkK_V(Bx!HlYFFsQvG7N!S6#C{63I3bo) zNbwTeM*p!$FDj{~KJ-jxgrvBb-NNhk`I77Q7Z-DCCX6_& z_{V*%kNe`sW&e0l>*Gc7BV1n9?rnYC8$YroNgwyLKJJMhhyHPQ>*MbDkv%~wxvTYY zSNw=wPak)-KJJVk`~Go9>*J32vF9Hrt&fxVvFjgKTOU{BNBE7}Txoq=i63+SxZL`< z96ur{sN}fyaU4Hl57)*Fwf)R$4~%gDaizF-cC3~yxvsrkxTJhas~EmBbwTfB_0 z@Wm>`gm8fAFei$>7!vm21O;*91S7$kU(`sSPO;DpWtb?&3IxjCfo}{;wpK}l;^rp)>}EF8BR3_ALP=WJWWzG=akSKKOG`=An^e%#Wt71Ptj(VowgrQn?P4R41bs$ zGqSnljefb|MFx6xb{9C|)3Gm3N1OoC8j+;xC*>ID+BZiyG3~shE zJ8t%B+^lDa+~8;oiT;P1B*x`v;Zp?Z_!Ob#>9mK&91-rO6cf4tq>r(lN65(3JqlI-A3+)5+y4LRZ^ zc8{Xch2{bZ*80&H8ITNh%YMoIy`d-DcK>|c9ri? zhgbVCnmT`T#pukA(O$N$#`8C=Y^Jha#Y*G+P5YMnH)UCYHvJfEHP&gJzbT-I)0KZT ziUYG@1eww7Y}uhTbZC(lk117xK}%$>V@etP$RZVo3!pVz04+((x~aThEQ=$=egaE4 zF_1DO_Y+EyH`HkEIbkE+qc}mDN7SVoSs}m zDAI?+Va4H~eLi*1z#1jA;J~m()_aa^*n&0v1#nmv9G0a>tTf@U>~L7ATT?c{fs7>_ zpb3M+q#DK43q&|r39gJPR=nV(gaZZb8|$*ZDa)jRDVl_MV7j3raEXY?7oN1B_t{OZ z{mZOxhMc0Uh%LIH%hMvZ*o|FH+G2s@a(ePmlb|%HX$wNDp)`?VMUFACdSI#dQYi1Q3fdXkJ5 z=_$4d%#HVqYIkMLo|;yeT3 zZ27a&t))RFbnjJjeyC8LJ*3=CY6^|X+w!FncB;1rRm1+41Efs#!(kZB>XICO9e zuqtGF`M_!;6YE_B76MKL7Ie!#WdX}zhr}ddP%KBHU}QMKEKi0mqfi=m`Gz5{8b8qI zaAsk9C;X<4VWwU(d%dNc${Jj(O1PJ;2aACntyozMD$^ZnZ+Yw_CMrbq%Fd9YbGSGa zo-t#@C)^-c@BzhTSNQ#A>bDa+QXTzYTO8;a7N;tBEN`4@EEBVzYK$3`mNjO(dMv!( zZlaepgYNvUw|ZN;jg>y-LSnj;ZXA<&GlL2-H)5;p=r1;S51Kk_MwPFi5%lCc?d64* zg@qcT^_^;v%Du`tOZ-Jic#kZ?;T7Sme#_2%H$Jr5M428^AE;2aNE`(CLG@#{6ublh zBjB8OKz7KuKUNHt*UCnymx26ErFVenI}NR}^laenQYNg0Vj$U3D`Aev)Vy8M9w=Xt z*#mWMZz=|5#zZqJmq*qEf*9b3;no0RhT6(V`3_966l5@C49Wa(|EqK|o(S{NKXg;p zAe(hnMeseyO&LtrAfr+UZ-8g|lR4VRE;}p@0xqCmbjk}67%1UfgV9KyTN%vL$;Tuk zj?CNjpiRXndb@r>QEqV(YWxdLix)B{TPIPc8ul$rUbWROQAZILV8|f3^+G8@r>g0} z41(;AKvhNekLclgZA^WazV7j_s37W?9hi^?_#ckXsVl$t;O^VVSig z)I&?s&c>kqj9KW$B<%jo+0=SLa_?Hb20DBs>CHO($=i7y{S&i+poGQ{(a#HN#y2Q>YA$COgYYy3c?PnV3GbfA4#3>@?v-~;|=O_Qh z%(o$%#`-#Qw~ecP4S-K4HNa-O)#g^D1ffnAiiBPB^Cd`E3C=dkj$HIO8iM4B;N+mj2+!ADJLxI>jCs}PUC&Tg-tC5DERPLaVO0PxUGDv0exCm~l zS+Ba2j6U8^R~tl-dp{Xahk=U{;lMLJaM_T^x!=IOsJn_~5(~v&lda9b#PU8Jl0uKL z86?ut7}qjpoau@($5#Y^Rl#ZyZjDCp55NRc6r!1EV-SAd0y@U@nYBQH`VxVugy7jx zu_}b5W_r5XFaR|yKg(2(Y>p~omY*?2z$}XJZpO*6j6D+!Kw9A@W^MB#$0@TaGmmFB z1JDGUv5b>jGxMRgN)9N9#RpXr=`q~8*(q6Fk&Dr->SsK|L$|}s{&0arW1v$@3=WWt zef82UzLscwk&`Xho9_#;c{j|8XmY{WGmZ<(IL0ah$8>HrLLxIenoN!iv0GrQl}Tl3 zgAajXPC`?ZBOTC@O`B_0JCyepVujk}T3d;TWdqaCHZq75OskK6xbhzr>qflRyI` zbWXzDs0f07|4V-({$ z>FHESEHHJ_Qh%9#)8TEgHr}+G$9lAIg*TR$XXGRXpJ;Nd*G_&_W=}2^Gtq!f^6CT6qy!!?3Xq z8-pg(RGcSgb+lsnI_fdwZ4ic?2U3z}**+k(yH-`kjV7 zCne5G=FPCc>6^qyO|-2CBw~!PCRaMim z)ml|ZPF0Dxw8V7VF^f<(0dwUzfzK7`cs8P^5fg3XCKqFp+?Xz_tNb;v7+^TGJ3qG4 z$&xgI1eT{V4P%J>yd-4ee=s%m5hGtrbi@~M?hptfyfyDMoSke@o_?dL8AurFp`05y zv=Ijzt3Sg+tq`4Yvgcs7a<69P#qT;69^_XW0f{D&C`k1)$Up)m{x~3!PrlEXSFYXz zERGLW!wU7MIqL%votc#`n-5nmYQDpv%4%cC(%P(9t{KG$(c_j-l39z(_?o=Mq-s$H zpzElK9~kUQjGDuGW3!7INrA|U9datuObG!}1p#c04G1Jmx2hPn`#I&Vg0U8OPLt{~J59|RY>yX9`h@3Z(nXju9xpy&hT3day znA2M4jwyQTR_5dt=p5K)3kZGNwXlUPy{s0*mP%fa(!V0&69}I$5dLhJvjL2Y0Nlvc z&zU;wBxawC!^3?bpT3e-A$7`n(dA&O7TD<2Hw!Y=sX>)F372a~~v~BrA0~0IU?%HW9 zY~Q!Nb~-7&U))(0J6#~T>)L5)F2?EDqhr8QN=saMxRWkvTa8S$K9HyT8l-K05%RG> zx<{1B<=mEV=Wir$K?>yOCQg>Ni4RXDZ;1j0)-0#*@OT*=O@IrhEO5P$`ZY~X@dr# ziazGXR;z64HiYC*mUVJA>jFvH?gP>52U-X(-v7rPiJh{2%e)P#sb^|#u4huUwj>G? zW%NrPmNd=zl=ax0`9`MDtN=GO$V#4Zf8xJ8B84W*}Y#(!-*q_L5`WEozd@RO_$oaq*{@+;v7z7 z*R_^vn#;nYw=vTXtQEJ<@1&cCslY-tJznRSqjxIPP+^yHGvWQGcN|B(uQ1s(ai z_3&3TY&;U!Qe0**2#HB3;3CP_aW9Abqu4BU3ZIrX4)Da(em&#^K=4qIKymtdB*Cpm z^qD2$Y_cSpCC1E>a5h;I&L(WVmV~p(lAy*;=voraCQH($ozSx+b2L$Xk2YBl)d62c z3FC}gI29IvbBVY`>`m^hMhHbBTK*2^5QG4zgx z4m!|DspocH8cfiIk;FDKFwfH2ZR4@PBj=)49eku%)*M~SEF?!?mN`c3Ycl$5H}7)h zfn5;_lt^V{-^DOA#n=@HKFHbF&)^$#Cd<&TJu-i74G=+Pe27X6H1w+8O9<* z56*cb+-?oiBY&4L80pbJ4a-kUSeq6m_qTR{X|Tw7$%edFCK1uuEA!L~XX(RA zk~RKI;giGg9mm31EUfAVEl0v4~L;=E1mB{F@xc7lcCKokqpOLUjM7)jYeUb928m zMxEnil9r>gg=H|wGi%8=gV$y;0=Z>CMDAPD!*8clD^!#4F7Xu1uV)aAXed#F~^i@-DZTa>ylz^>g#-d)X0V(ul+sfE*4bl8`&e!Cn ze8haQxIY(XtQ%`hgzSNkLOLCMx(&{;;jy>)lNDKnZdZDdMV}BdgdHtRe4I2;$aZl)DK;eSKi=e?2g%8<`1wU;5HHuNGLIn}wN194i zIK-bvTPY&7PqtEC8D1HlZl!!VmWk{r_?oOe-TyWTEz`8;6rN}fibNF4JUr4$G`(=8$tn`M z+m)hbZJAz(vD2Yh>V-{hv@89Myjt1O3tLP}1Cu8BW^^|;9|_>nmRuY3M#59A{&;QS zxn`n4FkM@EGh?R~t@7!;4aO3cz_FA-7^^@8V##0K=4dk<`;`*S#zP`0l^fyX0j33X zTgS@XV47w<5U0MY*&C|O!#%B(^adH%ff`jdJCNSX6xHgUI!v~>NDL_%h8 z^zs5MJ*2V>u=H@tN%&WHOTTLeSpH_Glir-;Md`Hc7NF;ifNeXwh4YbIKf=z%hKr%_ zOTr6-L^{$ot2x8Purgfu)0~R!70e}SW6m5>0cvhYifw5N&iM@&Y(%xZQY~K6a8aSF z3>OuudUJC<^LpnJTZRiq*$(@rnE6~QZ@F%kTcm}ui0lHIWMClIYw@jFy|inA1HYo( zqW1vmnzSddZXRW+aIRO_?*m$wlb0vjDdoi3<` z&LE^lk3eTbCIpdTM+fzd?n@VWMom{ujwGyw(Rwzd?O=K?g%cq07R+JJB1VysPM(rkaM|1U=_lZ4NR>2Ga*S~P1g-ItyxIm@9Rv*n-|$yN_zoYHdqgt+stw%e1Y`?(@A~E?^tdb@>!r8jSr^KiM&YOLmOoZ z;+ZKU5mB181OtYGQDEaj$|ZBGX#1@V7Fbvu7i-C1zj%?X&Ue3faQzs+?|uGiy*~S4 zUf&jWK$N^-MY?eQNtiU|y!otE|<7Gce9Llfl6= z1Azf$v`jbfvP8)Iq*l~rnsj1A@5ryCA$}y&M6c1}X3cP54d6*lIUJ;k{ORc33)!`K zkfu41N@z6Ww1EN+#OiF(rWT_mTb?`=b+oBY;5c8fsaK-dqc$gN5ckXJjd_9pUV8K0 zP2PQFYl-kvP%GZOw7$lm#L!{f7JcTJNT62TUTX{|`Gk{mr0 zB3G~lI4Bo`#G)$lTcs`1q9s!_DDhB!5RnF=OfteN>l%ZrG_ z(lYn5n%D-eR6l9GIS-9IDL8onFE>bP!drLW8j;@ z(xnG&31hIxdIiP;^J*_^CyVj|Vz`%D;fF1yvTt>RZQvy=5Wg&PfYrKf7(?0&W3X66 zBo7Z`NJlgy83Wx>J_H<5TC%S{KI@vP-M8B3dG5MMen<2{dU6QFQ=Ww7tk%-Yx8QSdv!ydQ`{QoU!DcYR~FM7+uav*8{0PT`hUq#soB zG|i5#p%C9b8?W7RYPMYTZkzRQkS>~s;q?~G&XorbKBF2W+l@%mB#fjuNxdG0b7#Kx zy5+8=$@(h2cM#ag5$wqH>t=)fYY;F-eyh~~ z_$j4M!$+E_fcgJT!lRZHV^IR0W?DJ>1HIS>#;-T)1rRC!u)!kTK$Lu#eJ-7cXQ@tP z@m*KiP(z5+!-z_&4BzZZriRUT?H*GLipuhBkE7v8D z?$pze~A|TKMFmSk83?5+8i}B8`bO+RGUiIJWB!*;f#tAR-F^kYFzV4IS&R4R6LN09wT!QvJ;eoGF%X z!|=4jDxPUCZR`a}>P5d87rofL;?+R7r<(xxF2~Fx>n&b}cDLar(t)0Dm$Pn4u_@R4 z(FZR_>d0^2UuxFpdZ_%1X|W-KMx5*Vs;J}OUUgK+BA~r(8%L^kN zIu!1XsSl+}zT1*jd)nm;?bBm1Sv|fd z_V{ir+Fh&qp7Nwn`3A&t#8SF}42sskNuV>gwB_@oJ(%F-Z(Qz^O|@uxz$~sMl%aQ~ zLprP&7&*mp=EefTwWY5v2ToOF9G|yMBhw{)rdBEKvlW5*_G0C$v*e6?pm+V*@8|eA zUkj^~{@E|@eNd)!L8e9)aGv{XOl6lXbjgKMly|w;ira}|rzd`Yuez-2t;X0WO|rKy zyd6`%MTw!%~njuw6ZV*|2y z=N9<5`fu>**YM#mo6Xt4VIy6>ck}5QF7?Ow3F6-l9;%awS9(8cL4*>5G0hk>Y^m=@&?sR3Hty69-jNp@{W~3 zR+I4BW5$2QE`W$=6Qs6?+hyU1V&)UPRSGo|wqRr6>a|_)LB}6O-Fm#x?t)9`$fGTG zwZH9X7vPz29Ce(y^L%9J9hr}`3VsaHkC5S$r z&2|OPOF%&cjvNa_L`?i>X>QoWp~u_2zyfKz7%jz~L!KIZe%$6!~rlOrOSx z)M7q1p&p{R6kcd>rTD=RPn5t)TJ(_QjO3hMiBnlJ9K8BMP}Yj^uq|IW^l$!)F*hji z4r5r{l?Rfm%VJnYl0reTp6SwjN1QLx5z53GT18=({5O&RZB+;;(H|ikcSi4CVV`YS zCQiFstO{g`d*2h%=`Nr-d>EWYU|M>^Y%d)U;f`070W^wI6a93l>7v&yFK4!#=+qF} zcml^+dqJ@@PRg*gY-%@6Th8Zksm$jjje~1Er$uB~P7%(0POXNxx^;RaP3mzMk1Efj z)^iH!aXCKbvDQn9ecY6nPC4Jyeh(Cv*zcu`?=t6V@}C>ET{b-koLK5Q#X-@}WxO#L z+BKdsA13&r3D{EYjKroxd}gw&VPVR%qvWfWO`VxshtSyMsXMmgsWU;0jrAo@oeH51U`C(DQjM)I6_+?$wMLAk z8ovh7RCkM|a&T!;2x-G$IxUJRSbP9T^A||ZOf)HG3m|PW$OS>VSe?#FNZYd(zG5c0 z3*a*W{1p-4ya?bk0le4>a4_;q3GiYL;IG&o;1^#oz=;s;7BiEGr=fFv3JCTWGyN@5 zdubtR@48^rE@soB0=S+H;6({=!|)Q|FM;83gD)k(=P~>|04GXb3-kqGIIDI<@nQ=7 z2;e)5LVp3caU`8MQOX51FWG(J0EcWUfa}=+zFGntuHhJt&O=Lv;j1NvBe>9^ilxAb zn*b+5qa0)@qwO%^NQ-7mb85;i1>A#4yi zxlq_BE-Nn5^uaB66&E30z`o&eL0^xyaJ0CnI4BOw4-R7SGtCFvZe7hr z>AfZ$6nrN-L~_a^p%D?Nt!{(_jy0e5J0_T3vPCeRU2x6|uXAgUwrhQB&MhMZQGr%? z&fpgnDq*PqDRv5IJ4GGiEg0&U+!BaI(5}SJOT_jiKU$!JNcQb67KxR-SR|hRQj5gZ zmzNTWb<(I5wU$Wi9l1b}SP_#ur+Y-TJ0%%M-HD@oBDU@n_3sfk;Jk7SH+v8+_zUYb zl}rklSg+xDDPl%(6m+e*ag;Yqz(a&lRy0J3RIWPCD&g*WNqE<77}Nt-^A(&eAckE2 zp)k7@Za@>O<@kvxzXU0DC3? zpqHlpJAb!nLO83pa4?)}rM}nP0<-XVGxZ~bzI;CVtGLtZtf6NIm6x(ee`VHPL#5Y$ z{o8C?g(iJ-aHi<4rMIui2+4n|xAFQ1)A$) ze*pfr-ci-_FD6uc!+sc^Rc$G6+P~86Bx#m&rX%}HiN*m4I^~|op-;TsPSqgd@om!~ zX8mPmp)6VyG<`e&#Ht2wnvHRztYvtGcvvo$yjb}4w};2xPLlT1BPu$oVdLX~^lQ?m z<>*|#I)9R11zCMk?}MxJ$Mwr8yni}~xXs29;%Fc*P2mY#D&vP|L#)sHi{VW?ockQ^ zo-{mZbMj^34!NjGS52$|h`=*K$}JL~`zzUWepna=bcjhA?T=|tJkcooG5*0Zaf#il z7ln0ef0gdb1Jd&DLQ;2XyO~0emNQjKB3ScY`W^06aPw8!UG|K%ZYY`|nlOI;v=fGX zwt6V8?{_LI4AZJpKv-SmPv?u2*&UNPrShyA1BLI_FQ{)Fi6q$Q2uipQUc+PH%v71; z;2;4)(#Sd3-TK78j}wnzUTV%`IT`k#BEyLC4MOE^hMQp1-#;5po|n4}b4+v%5L|bs z?wzOLIU-)H21dvOYl;{fM>bwgmaFq~7AOY*FWAul!M+TGt!30q*8~z*Xn1cN@ONX-o69OC>6G?S(Dh)p0;n_jt_km8&oGeshvvN zP`P9BIRXB{se*-ztKwwYhAxN}GQ=+C@w&J=zd^J)7Sl0;{yQ+8yvEg{L{w0;s_6{ znrGqyh~>B)Nw9$`0V;&- zYYp+(5-w2UX+SIkWnu)0Qua-Ez(p%b8k_g_FuZ$bd&`iKy&f_$ zq`+Pa3{ixKLPmI#|Mfc7@##g)kP+TT<7E#SA(MrSSe`0m#Bz*fFx?yKkdeI>GGaMB zWF%&@kdeI#8M#O`Hyl)V$oW)C9HDejeWaCII;fs%rIrq==j5oWYA794_qS4i*`&)O z;ki}~XUv&(Bs`?wss=4Aj~bhM^*WQ4hUY)4&p+XT9i*K1v!Yh^den-6p^REFKtQfr z16|NM3EaRaRZLDz_o3pa8&ZZYMqAu*y4Z6{6Ut>Uw|TI6Oy#xmh?$0tNnJ!(9Z+3Q!7Xt8R7x5< z+^G*dChi34vAC1jNY%y=mLXrJguy5S^h950-EWmggO}5IkHmBA^{ zn~I9T>DltPEpvKpG5$7o;B!_}tdJ8IuVXa)yafsG5~QI~c*G{D46jj-P+Q3~F80X@ zx0`=X7sHzYA@tTgji9Xl6ywv=J-K9iBwz8EW%uwD*Y0o0*OpnxIt;{)TwW6eJ{9^ z(2aYVe%bd}`MMl}K(Uo#ptkl}TZpG>Yeg0DL2a$H+RD7G;o9z-3$R<78#GKfq6At`3b=!d?_LYNMeAxvk7jkT`1s?kfg|?zt4?^q zgxeTvRr6`diBIR~Ui{uYoT2YqP|b1~R6|e|+-O;(ye!uYM@N@IHLHA>Ko|6u>}@MI zDR?D!kONskZOc}qE@R)IR1SW+o0(8XbVzN~l;@`zRkK>UYG%8ORm+1GS?p9)%}`M_ z$czLND5{3C1imOg?a0k)%_;n+>8{azTHY6ng>ipP1fzwo46R49Y_3*eHN!Hj21kK9 zs5%&|oq1#+l|eX$)c_bf@Eo<~Dqsl$B0C)hmymsyi#??kmuWXEuI4~+Xav?6URYlg zSYvSWo|%DksTf`l!yKN+Tx)DCEe3{=iY{=UUC+|zoklyinB?)f(d)`2k2!6DIo?uj z0?X(WxVmu(^W6`NVT?`@YpA{uowCgrO!4zyYE!&|owr7(Of(I0x|Qy~vPI6GD8pnx zIEzkUZPYm71q36NOoQSUnqF)Mgd(RC{)#!tRq0;0U0VRS za|-~)3Kj#=zx=}jvhvS-fq(=I5tQ<2toi^AqG+GS{dTpWDxN0;^CM!*I;mgE^oa#- zOACN$VW2UMtxh3rI}kfb|6u`GA;aGZfM!>E+71jGi@9hO5_`vu>8Oe2!y@)Nir6C~ z@O1A4OAz3GorOSQIUrog*OoG-*3v>50<~Av4e&eRA+`|D=OnEPfx4*5`AN3OwQ>r@ zFZ2&3DIa18)XV~)W*3XH&MJiSV)JdS*jhtYx(`K}DMWp-7%H!T#~sC93zxd2xX6C@ zLyiZS`~wKJD-}ak$Al4Lszjapa$xaI9-!$wKKj3S!O$$pPWk2EBF@UL+hz3e`o&c_ zppH#10oULw*w8R3ID^%br9AVMorX`J)Ek=%7XvP<|z8#!f z-!2nbfgmERW_VLO_=Ba*7EYF5Eu0E(s+Kl14x+2X`@&#)nPm4B{bk`gsMw|+v1oEX z`s9~TZby+VhO!)mE7OA!(twX=c^Qj?^w}Utl$!@J3CO$%$Oi>vg}gO^3}SpKflSC* zONdZl&>|qCuxh1_5jN)ovfa0H&=#1>6w(BoFTsn>U*NnVI1gI*T*5i}bYryM_HaH3 z0y{2Ok z7Q?=X_)OhjN;qFAJ{!lsU|4g@4}q!?r(wFln#(kzoj@l zE-uyLc7R2{WG*GjFPI9XsrFQq67dX%QU);0;;JZ8ZRJ7B&6kwgxZ{am6sYoo!WLwhE7n zCGY<_gvNswI|(e9GR*C@9%B*XEb38@0ccaLT2rQI0noPUeE+yrmY1vh0fvUkGR1NM ziBAQ*U2q8*Y$`=?+LX4XvQV}#zEG%8alwEz1w}<#J3Ri8s4g!TkR=#p@g(Z-+U4EA zPufZaKRk!)Q#?u?1(^=~p)Y-LE>6E%etcB8Fc76qrx zq*^Ix6}3RjO2#xE!Q?!O{&=QXR%w0qTMdZm+EI}1mgxfu$f5!ZzzAoD>ux0` z$Z5NPPIyJzOt)>$Ewi(2&I_+|OT-*lz=AW^qb(t(2AEs`dg-!uE~2^ zsihn9kyZ_*%k!aDYPqBSo@VL|8K)p{wlJYMvrOl0Av5Jg@)fzCfoDmif#?t{KulgA zlF18-U0{VOljKKUnf$FqG<>3yjyVZIjx_4gX;dINl#6?aAKX0%1AYDu_$ym7c2?*V z0_FU2$qDjzG1oR^V~v|o-#+Hc8DEJHK9+9H}mWh^>2sZOC^DzJf%#h_Ng z_uX~hX4WI^E~xXvSFuu}6mn`)0b zb@XUfysjHQsVfhNV?_RL-yS1a4mV8QSL8h}61BmA2ziAY!b#Vs^4E=xJHpvD9fT#c zE0Zk;>zHoS>5o3%IQoQhWOC<`j#0RJraH4CIg$*_nJ0A9)f&TKD67wQ_w|#Fac$vmW3#;MZi7v?4lzZqNe27? z8tSABF+C;7MjuYooWn7JmK%W#inENWIpdO*jb7WKa|J$AQM|x8g~H5Xq;jHCEr=|T z4q`GwO4URu|~Eb?pOI4+R#qqzTbJV?(2EjYfnpMKrP}L!&PF9F2Gq z8sT~2z?)5Z8)>UscDPIdT-tyNqRn*_I#;t2bl6v3BOEXD2*-W_^1deNGP}LkB>0+} z*!KM?XP4R>%~FdJ)jWCa8Lc~xm9xfW(~)R}_(}Ls1TQah(_^RRV$>q)hdyA!v1XM4`c~v`jB;TwTY9_f z`GxUcM1}z$Y0lpQhG7?uE#h}Tt(sVlf6}_&U?XcB`^Wgw2IK(A%&7XlI3-)TKn&(r zZ7BjUr}SEZd$%u;(Fu-k6n1GQ`N0E~jJ_t(lPHv}1$vl65NCj7Rf+s{uSr-pEo@}; z)#oiIaiwez4pg;wU6LIIiFA#pmNYd>qs`0345?RHDhs=q6Tygv3&z=I(4*&i>A>&l zOKHmG6$Z(whQI^4vQdJ=DfLW@<}thYMbeT0O*R_mMWT~IzQ}ZA*Qx94R1=v9qTpuW zS0(Y*J&?b;TaQ6x&6mcld-z(l$+B-!fU59|D~qb4DO4tj0*1%WjO7+_fl<1QA+i?B zi@Uz+4mkS~*jQ9wm85=~sCb&)nPZ7rOhn%dnXZBFNlWW9ngOC?&e2++-#M*{=Tc5v zVkZrtF)Hmzk>|`+ZU##Y44JqVa7W4jOhlTvrHCDcLQwAfdt2zAWCXpF#{;+F435Et zECzm?Z!WhjEVo^S**gExwA|h^rup>-DF(H{$cUE$U_g>fieJ3-h5j)f?RP!gy4krY zOil{)ME3ws4Wt)dV5a!0(C%nZ07%sqN1Y)tl#RFMq`Xb25BthRR%Ib;) zG=K*@)U^YG%z~Z)Uy#v&E%9GdfsCo7@=$$qm{hpQ3Q(gX zHi70#12?&g8!&;LnZwdHGN>w>Tuz%QD+F+=lCd&lC64w-|DnU}nDy~)QG!&j>9@g* z6T1_2PCR7`)gp{~zVWOZkqjBzz^FB0e3LGi%RwW$sBwD5^}lx8c6r7ytf~X_A7+D$ zP{i<3fq1ZkpE5>xj-|SJVY;qYKg!U#M+{2m*e=lGiZlbkU<8t)fv{x9%mUlHlI9r$ zt>_@#&Ta0Ty1A5kv{$4Y``{ z%H#9`07V=eaB32|*zm3Xp~Sj@owQF-?^~Lm@G!tOFz!&{)BV;b-N?nK@75>6UEfkw z#~oq%@Gr2rX!za04Pi?7t}3DL3GeU(<5Jf9g~0GTxE=Un>|5Z)+6-Spp@JHPe?n-` zJ0Q{sEamBBMK-!MPpiYUw~DHwU+`0m+{l!rVYbwm7wIp65mBU8UWkKqoh4wwYUb6# zXVcu&Qe5z$NU52$ibztgWyxZo<&pR}=?E>!GKp%*7I|{9w1}|$5>jHMA0}yTY?9V1 ztwBI(ua+|CS(Z{!*qTMBaWzEF-stJHK9QL7%=?E-^tNg$(6?JD35lpxzf&D9C3@-$ zWyG*iASbs;hz5H7BgrzmB>5T7(&JNR4D2ieossi+m!vnV2kIjzh1mroFoU8EvlM0* zWzzn?3e%9xj*b4hJ<`zyjTGe7S)Ne8ZjT4N8tG`uNC5?b*X;Q6aSscwSJAln*>>D! zoBewj_h@e1mw`rN(!_HOVc2s6gfghGhX3q*&l|EP#qh;F?-i z)wXSsv+=Nklog4!TzM_rYj~a11P)HO4N_cJ%Ci5Jup1-*TiU54rEB!BV6N-tBh*3W8Ig=LxJ22jJ^9ZpOpm&x3K~WZILx4v7B}Hi8b|# zFd(V>!`OYUBDU^xWmDOGG+l9HDoI%{9K5V!ptCL-DZ9_-i~|Pv%&B>TK7w8BO<&#D zg{Gh@U;_Hc8>)|xR~1+;3zXgOvz&MV-G1oZSI)Zo%VkA!>(G5|3@~*(c7I%VpCb~P zK;e&#z#!bW2{5eDH7qkl!+DpoR);hK77LwPNAzgl`pyZP8jp&=j+f&|RBWX+4vMTa ziZCLJxvNH0jVF3Pmf%8c^dT^0n_jJAz|MMATl{| zoM|2KZ}<3BNovMj%u%Yr92AL(8>8RuroB>_YM%KHp&{QwZ@Rf7gJ|I@Yko0=eh4HG zo1rfIG`UM0RWL07xIonn1ECAVg^gDCSsDQw@u}(=oHq%tk=57eg6}-sjG;^2?1MBK zE16Qyl+kqh%0fSR5fH1Bzu3r19aigA*!z^7u7a|0r}tdZ3lK?1`7O5~)Qw*@xCB zj^?Vwy{$x(qpwJJw-URab7w12LITAnt;AFo?Ek`gZ1xBfcTodHQ{_hRMn0ahVE@ld82$zjBjs2rH}y#eMjCw1`zXaJN>^D)pEc9Z4#m&}i=P9` z4YbW+c|~gZ##BqBh6Cy#3$0_Do1N>}rAudJ#kj>-mF<8Ls|rs(i$^lNCk^-gg!#d}VV@#FR!eIz5!7WXPqvfZ!G`ev*MfFp~gdfyN7xRe@;$bV_uDr=`v%bX!ok zN}ew)6Ki8^tVQP=8x|b^S>(DLCzI}40!yPPG{7b9@R8XE7ZTKstAXPxkeKvO&N`># z!O_1G_Hx1Ade@|ARKQNhYj1EHci003LjwY`TxTvvwk)C%bcm_lXl9>Q5-3o8Bd&f1WDxS0{-bJ z4weLIZYx;Wt|M?zDR(vk4u}fOhB6!iBZfa4$y}C4`Bfnp8Sv22GvsBh?i@~blxyJT z=eaaVYjdm#?neHUyvVBL)xsS%KmQ`Oxisu6^GTQ!y5U{x@MAJnr?QD7EKy5%K^C0y z&Q3s|hi958T;fpOn1bjO?NAUq@)ZPO#%;>%L&9*(D=}BUMSdjN$0Ww@|I%oUm~O`(;6Nq z>k%G4Xz+VgQtCW#bn%)4Cw9^y;}j!lmOhWIeG^yB943mSL;8umzSRa`Luq_d=Sn4e z^y7`r!An|`B~Pun7jnDY1~p@1^WpQv>SoeCY)OckdJoIJXF&DxWO6m20nr!?Q*n8c z@D7+W5jJ*E_0!=@$C(pSkv0U*yh6FrZ}##Lr{Px^b(UjDDiDUVCa&g4J!j)d^AWBn z&u7yaJlDMpM5~Ld!A(w>UTZ@zbEq81VtfkK%IKVhY`|e~np);1p8Ig1rK!jttDGze zSU_2NR5CKwG2=lCTb`!2YlvWq2c|1)+EfVDv$!!Y9z7inYh7oLUNK@X7&i;^Z?SUo zso6yP!6xiPql`5m5DBh@)Ki+XgXzX*FP9iWUKxGt_+m6(sgzU=DNve07OS>ZtT$y% zr^k{l?BRNrm4a?cP+_zRP9f;RN7fTZhR6C^fdHc+_01e-z;7|SBFf3sAm~27aBTR+@kUEqdHQ>Yb_=1@k;Bl@FomwiE71m;~ zVMDoU@)$W7plR?OfaDH^ZVCJz-^LQftfnPFNQbAD!2kc*dk^@iineY1?4}nWbP$x2 zh!T)gk`MwY8+u223t^LNl1*nf2?;e+Q9)3!peTqaN>LP4M3f+kGyxS+A4L&Cv4W3S z5QYDB&78A)Hsz5=lIv;RGq9V+&8s{bWP}G+{q&46X{(*`PD3~m5rU)+wlSL)55;?-; zP6bCQ+znmSER9mW$OaiD21oZS@d02jOFQw^8DJ)X0Q5oQ78M&(VP2&)^mvg2K**;m=C~@36WF4jR29~x0tdM^e@33^lxu- zz-;eX7yxEP3FG~L5^?axgd17Hyo4Dc$~6BH;DUMPrE$1yoXZ_F7*xvT1vYib0MKU= z7s`pu_!1@d2^&nw1z$Q{BDTH2Y;y3-ugD;qFjDcp8to-FGEPtDIYjVb@&k=!2BNu~ z2!%ihu&iB(HHV1d%(-a_3@^~zpk#>{&M13Rda{H3e;YIFzLV%K@tI3H#ed;%cf92x|+4`)1$nAjo994uw&nTAJGPK*LG6+ULe8z$=E<2Jax zrW7}Cz&niT6}~i_)*l8w1+ki>ED~BU=+xJ;jda_#0;(LZp$Fz2xFME!XoENfobyH% zfEVh4`!g|yBrL&z-c|TYvO}N!I2BN^2z7QFX_j>!LnA-pa9E3pZ3Z3m_Vp5X8Jfgh zhUg*?2M{6xYCMXlw^^)u#1~a`lmul!iGj19z|IP>iRR0@O;9x%l|Sc$7lZgL!nYv- zbM~K%wIxew~I*l)~p+#Bkt$?f-geBhN2R+iTl$DE? z@x3BJcLK1cTuf^i*I%f>&oZZ|AHKsdTlHCpn=pjD_QQLO3HD>iOVAPl52Xm#tPsqt zDDG6RT+BPDgVG#%v54nX00nWKAV!V}BiN65^d3~fVM5%JiHp7iw^V%iNcEqGuFe3( z0-4x5?;Q3R6=IwU2IPx-8AKjr!A*X+#}A`NGJ8b45|^{mxNX2KEJUyhp|*zw3D^LE z29yCH#5e>~LmH6)%$xHkeBkIfAXj+Qy^Pqzf;FNORpdqBHZYH3H5>G)8J}Y3MO^0D z$qG9v@km@`${7gr5*KA8&qh05mPSBaF%~SVnO)2xI5!=tP!!A#5XD5`rYgBE^Riqw zMQV}E&e;a+L^J-le?|DH0uI<@)^JaFBj{PMy9{%dM=ptb%q2ZiR6oj1Ff1S_SRzYe z$wq2JchCUg?JN5!IV_qYjt^K$e6G}oWtw<*bV5nVrN}oCGP%35+(%$l^ZdxiYXR&OB3_SkK6E~ zU{O6a>p)(~B;|)(bAzTiafASqyi}O9xDpVaGLXTfSZzy&j@`0(4ad)l@0e(l00@))rE8TWa`aI! zK5Uke1V^Diib=s}#I7W8nal`uZ6;tidt5X!GbW3?3?_K60h+x6^Kj4t+*L@*3h#cn zkqOX%hKUQE!0XKfrNksD=u^Ft`Q1#Y3G|==_Vf54$j@7xVgaj!h!XZ}&JP(bB@mf5Xv)P@ zVYFlOft!=BnC7u2Qnb!!fWW8%Bcv6WEC-rOC~m=o+y#05&2S_Mx5yI6$(<@nXwZBm#z#uWW)+VIivF z_C&0c!RoPqCg@ckVRQ~)3)t=#3B)?roQV0Lx~u8lhq(1mX8ES zSO`d%w~Bj@2=l?Enkv6`iU(Y5BB7)+0RkxiD>}8oOysCX4iE;RPgOXeH&T#HChBKt z0xjKzLqdu1v*c3*;gWPou41-)i1L1dsVZg#P~uwH&vGl-5*X9KR#=W=iU1EEAr8U< zIaA1mtMk2()toC7O$b6UOdJQ{#*vUqE*dha+WJBW60xOXps6q~T=>59lE3JBcSDtB5~1N~3i~uHr<0L5r?28i<;HE?{k}NE05R zkcl32Uae0O0O)^T&Nc=Kxr|;yF=jR3^x*UO?}LSg#X35KrAH3CiOKcZggFx69RQg{ z9)ND>D2WFw2|`pbWbPV3U+(Os(C~pX01bma3Gxw(%^9XyE*Fq3ZWUIbCjfv-a5*k5 z*sgi!RDj>PMHDRrw2!1|4WEJ1!_2egqD8I}5@F5r1y%h0xezwq?%so_7Ck4VaMlmOc zES%3`zWfLQ%9w}OQ#^B{?}z8Ug!d-x{jaEnbLRgb@WPvv*@bf{_d~deMRmg@f5bO~ z{Vft41Whi^eLg12i~Loq5$uy2g>VT7StIwL!p$uN`;>=cpsEbQw2J9OCait5d`JM# z#3oXBpH_GTEJMUp-r5RF0x)mIf?XLG+JMN#rHss`FCR4od6>rl&%>7$QmVuT9 zk3e#5jXx-9fY4&|R~q12eK9D(Rmgm_Zh!}xfh~fXEOW{R2%`f>V3}J6o&d{NH>7r> z=}8+A3euZ=Zdrf9Kv6l=<;x9FQjFXZAEGtRp~(N(4K++0@emyfmQxc<70(;ZW08dE z@FN$a*Ot&V@BrnNutt)QiNRd{^D?Lg%vdaPhozj(y zyc;);RmrFs0gw?JY)r2Y{w$qMESG8I<027TYQi?c2C*LCcYp{rm3Cbzid zXF!wXpkzxJ5^4qel7<^5pdHnR76DUmPDCjIom-i?@fjmyv>g8`1q*w*$+ zCbUdgLdd^gSUc|WrVWj-MbGw3rMcUWWbBi>c~SK$8;nLiRuW3ysp4xbQc#Q47xAYp zaZ3qW;)V0G9IpF6bXDAcjGvi&64nFWvMg?37EF5~67WHp@$|ZYm2)f{&hVc1QnSct zM6uq&XN|zf@L<_~2J__F8~}d6I4K=SVLZTysOe#LgldLWIL$bkJ}Ixqrr9kxLea0{ z2v$WzUce#NzQO^Rl!uEp0(uFkzGaohS$y^1~oX1u_j32@;!SG)L?X z19W$TowDC@^MU3q6aiszZJ?TIO^{Epj%WkiSK|P+#3wxyRd0E_hwxKG7bGksP=XpO zmZKjZu23wURF}I~KMywuUpaArHrzsJV@3raJk^&g=XEDg7LWL1yL=QU6hd~8RVYEP z2AX~065yU`a}Pj3n3-S@o_0f=Prssi@~pdwDJ0jd3PLQj7SRuA&yj9}QxM`N(%wqQ`-q1r~d!WaHIA!J05mA&K zK^d(EkD`Hsp$bO=NO*33KoQOXqoCKH7J*lZ|E>hvkBBXAtU@Rkz{nvNfUGbEpcd$aI_bz1 zWB>&PTe#MM4Cx={3lptnk?=Q0FIv$EAddgx+kMO_Z;`{!=si&6{%+1N_La1E4H)8XAe)|R&P$1?nV$pb-ORa;s@T`ir;g;VjDl1?z!754h)_t6H zAcE*%+eD*$;5?%Xl*`Z_p=M(mb>IzXg-vaWXfYQp*hji~QbXp(n^zB1fS$WRjb~`~ z6y($fK-hp03fDLc2r|PkFevvh#9+Pk;V~D-9d0>X!bVZ8OFYY%ZDl?FJR<}Oyf9C8-v3BOlCZntwWgQd{$WcTP0Roob zVwMx^O6KxnE+jN6hU@ux1tr8pV}@I!Tm@hTok%w29rA?K8$^{AA*d6UP1a`dvUoE$ zeAnHG0vrG)(Z1id7BKen03(2EIiTYVo+UOAxCaeyTkD{8gB&?pXqfZZoa&eZ6>8hR z$q8cF2d*;0f>RbCwwr_M zzaEO3E> zrI1w+t9a6H8F_}my#_tPh8E2SRXQ%Xk?c77gRji$U9YXrFblkt@ zj@R5pITYAtMWu3hdbh<&MCU))4v}jczkyhzh1H`;fXo2TwD5zn2O$tS-=to zo2wy@6G6zb#AO`;V#2twZGk^Ibz&Mj#L@)%^pwzG{f$?TWUyA@cnS#s4}^t?V^6{T zDC`j8EmPI{^YMp$RQSsRTg3?;q1uY=YF0!~Ro0>9Mpn)?kiccFNB7lw*!slI1j4{p z0=`%w`>FLX*|X(ji%oeUPB{ITTo`(Yrch*XRGrg_yBg`Oirsmvm$4tDO)z&3VT8_s zykrKUA_+jrTi1xLJb&avWZ=K7Go7g}KB)Y+(Ef6q3 zLpSb5|99wy`5?Ll2*Ap3K9B*`ifqA@v#9_sII_Sup)G#EMOCs- zN-2dO6KsJW_^~#@55C0`f}i?gM}pt>k@_4y&0;w!Sd;Fn02YRKl^*ZN9oBd-Fz&u? z`I9%*N_F#CqMO`%(!|2D8cO?0L!xRVMuPA*vHs6GyjJ)pgD`zSgzJIkL)nTnEL?f&HZy|01^u>vqkih+k7}QDSO-$F4*u6(QJ-h zmNq0pcu;#=kOKD!Xn$1fW(dD#n(Cliam|66$)m*Lfv$eEDAH{ea8ke)fOVF4zBvQ~ zBiIseUW67(-ex8R)8z{|jOQsq=Y4VHS#Wp7qEbl$oWYdP0z|RG7bfk6mWg^3x(Fj; z1&lAi1Pi_wt>BI#%hL}B^+og~qn49z1@;AIJaRs;@02NcyX;Ba);*;^A1jvO@sm9H zT!H6DipBGm51l!d>GJjEzB!iPz8EPY=jSE%$Ecz1qH|_cz)O61kth3#u?h<~hNhY# zl*iu(8iZk=9ITG({no<*qww(e_is@gvXHh2mbruw zo4^4vus@D^4(xXkHx3Maa}DfcF)(wrfgKYoAZ=hDyzM!#W2QP97+WWT0UkJ^3!w!R z!Wb3I70?M5DTzuQl}+!8LCGx_bEvd@0_4An>B2TX5DCZ470YI3Bq}>VJJ5G?L|#j! zm7-P)`@*t zELdbGd@M_tQFa1Nu@hpK+_MuvpWHA|Q9p?W?1T@hk^li-ObKeCShn*(tYsw};Hm5D ztHv9Jo{v7899fn&PEeEyhX7Cn7AyckHI+h%Ee(2t>`-E?=%<&s85U@QHdHUVv{Azq zx8Ko#ij6X`5$Lq7gwq0TTS&&LSkE^L6kV~;?(!t;j=51piIv+!yW+rn50(&8o#uzq32%~+}znW(4Z6rjPo zOQ1*)IKow#C#ZaLX-0-tO>p^_P07JBleGIz5iJoISMf6<4(^uetFd$&At9BhH>IqVDMJ-@1 zK?+?)swO=9v%)wUE>Eu)I)lZ1&=G@%GQ+V$?IBva3@oY_H3@%Bli<-#EJwAo;RWuh zus`RcX(GZXMEjJeN?1r}B7#~)>&#)7R-s#JTHvj_IbqTWv1|~Ei@Xw4Z|*)Xv}f)C z&{UYf_{}|_V0?tuEZ=w66pZhz<_xvpEhKYBsBHTg0`fNsw&iH{qb`fI49Ma(*zHAR>c-C zCv=0Ibjn}0b9VnQGroWcqO^Pk;NctK0*_qPSyYtf#fdqVYx=iRP2#YVQQ>QZl0Uwp zD|`WNg>TtmaO3}^@VPR$6h607S1k=d&=tO`PZJ8Co;?h#wz|S6QZ8nuqb{VwkAON>wsSaFh;SzPI^sb|JQ4P4%V?oLu8Fb(p0O%_Gr)eHl z)eJyUnFc^vu>jyp+Q9&Hr%Bxb@a-S20)S0`%3_!EM^J^%N>-cACbxCH64rHpje9|K z5dYzDOhyQ&B^tPK$ePqc`hh$F7)WhrKLgOYEJTrT^T#p<)3rF?ikHI736{qNq`#Z* z!FoZrQE9Q}?HK-ijK<3&F4pL;8y)f1%^Gp9iYTU9wyoi^&ffwD6+eAQh`M;Ip$$~5 zw-H`TZGWt;UR!IBIRzFCl`6KT06RQ|>=!G>FqvrC#T{S^IB+#PQJn&C*9`mQW3-E`<43Vz&YwTc9uP%!95r8Glt@9C7&`%nPR>#Z8vD&V=?H z%kO`17!Wl5TXEN-$B=qlI#VSNhWkkkS;Wrg8q`u>c*fB?j(=vc;a;smk$V|(`*I{%CQq@f!EwF-?pFI-R>QK+BdQp-pQVlPYq!uKirB0vmV&-GmF~}OiDRtdN%$%G zVYM#NYG6IMH86+RL-DtqhrbE&EqZ7@s;)>|iY91=7X!n(sp_D2A)U$*tbkAfYf0wu zQ67YYTC5(K;w*kls;{9s(ffD)>!_KvlsvB4AepHljRw)> zH0tR9LJ7S~^<+KGP*26?Q{Ju~$%-J>Q4x?y>kgJv><)9f`U8V8Nda^irge&+Gj;Wa z*FuAz8~Q?h9F<(z{eILyy+=HBq1VVuYFk(WL^&KUBc51ZCnpVAE4RRaCAcoHwLngG z$0v_>azoauUVULQFs+wP0Wz2Di^v1(j~av)VfodvQet3d2J0N~SL|0Zv*?^S)Xu!Z zbR_QcqxTb+`dHnFV&Dt7ABE1NJiZG*LoZ9>z*f>fsSB4|W{c7ngAnT#fJ1NxFH0A_ z7ouiU@@)9eVI*)fKzA{KWwZ>CF@f3omUg9YZPZ$SRmfge)6)zG6c?+u6Hu-(E?QM=5 zT$&qT03F_HcZ2glk}xZXm)hOn057;lFjkJ>eY$mo&Kga`yr3o%&&YB}0Ivnk!|;u} z^5!#ZZWtZrg^h1!NUqq2cm=s8UG#weJPPh-@o`{XD&iHe8)B*f`k${fC-CK^$X@bN zWH0xn$k+v?C>7Q=xJJQxsew)a3`uA|xD|CpaljqD&JEQKO#zl`OJTvU?`>90W{fLH zTJ?iqBgDl-rc(Nb;>?OLV+u8Z*LRSAi=7RLEHH=iUN?E?n%vRlf}M6FENO9O(v!&p zGT>rduHrKjmc$(BWR*mF`tP9x+(~mqg`tWUkX7PBsnTfUit zUJK^}v!>3qAVRqHHE4DCeu}6^szJV%m&HasU1Ovs1#7XOR(By)LH#w9{OT7fH|~-! z?nq$;8OJ94fN(BNQ8$X=G##lCSWIyVQ!KWeAu+Wf!Ul>9ZH3DUznCN~rpOY)`e1&z zLK`$$iHW1cjZA3X)dY6e^kHK!3ucgG!W+=tL7Gv4lb;9`akpZr;3)@PR|QJqyuM|b zc&A%g06dPqi5NYr+$wAU?kYKzX6U@GnLK3@J~EbjwU1c<*#4B+WZA8NFlK3#$C&(e zBPLHxz#EWGy&&iL-7!SaquKJMAD1O2)x6n)DHj^l8ukv3L%WEc!h=*?3?8Py!x!qz zSwCDjep;wFmbD_Qmmw>`L#8ikS}&4KhGeslYI;?WSEvh2K;(Yh*ly4^lZTN-!&Eu< z5ejT-ls4#3go5k9aFdb2bwT06g0r^IwXiU2J3|sN1zK93c*qmAgsrxEMaClUS1aN% zGfL28@wFA`{vc_Rl5iE_Jn%Ydu}|&g`g#)Q zu-lh|xTas`rGn`2;Ugd&;erJ^{irsFc`pP-;9V|bi{?`kRJ|8UwhfNOw+ll7JSq;DRW^k#qK(?>bybFn^)^s1 zTl8Ke!JxIhD3nKMsQ7r)TVV2?g;zY6c;!lid5{V6lmQ-nXJPbSaN`Wa5XA*5NJA_D zx2X9*oZ#q#>>#WYIPt0aj+B0S$TNQi9Ds0+Vg~TeGz-XNo;70zhY|G9srnGeyxlVB z!e{jr{COKfx-~D)mZ44_ZgZC86(=Q?6ijtki^AJaRtpPMYk@ksx5F`6Ew$#A*eWZP zfWngEw8D~tj2V^Y88a%C`f0W-dx6@myJ{=QP{S*|`B)j!UY(k4D^Q(z_H=temO8o8 zG#Ra@l^5GodqIY+Os#CD+MOyYQwuXyQ6XZPHD$b!84C)FRjZnwZFR`ZR!4bP)n3db zdx6uQVN)yp)o|ADR4aYkyQ``+-BPIpPRO>}QzoPrmvygfouWEQ3X1LdHZ?uZ>U7#O z%T@f?ic{?w-76dEiK!S)QK7w{xO-(ASH|uRn*&|8rDj=+ZBwn~YLT@#yL+XdI;F(s zDDUps&^0NlwaA{DYb#f6Wkq%eV64TvR|e~^Qmtsf2g93+($VZ@M3JK~-R5*g1f129 zB4@F~X3fX23L;|5%2a2uwYbDtrS_I7sy5VA0e?d$ebwUfB3pM%X^kZ@Kv^QMurRlz z$a8>yQubQCY)OH&)N0SOrsdfTE!WZ_`brFRmZUk;9riTiR0bMKH#X#ugQ_}cHAT&^ zJApRo+1jWx(FxR-o>$^5wmFClHV1w@CZdh86ldgW3IYGBGzCN`F3hzRbgv9>x98Ci z#_2AjP5qw~xlspbx-~u9re=dEt(h2!L>)I}HPGNxYI8V&yj4cn&a;+$yVLnss%egD z*lqH(G;4aU3x5K^HZsan^Eh~q5qIQ>9k%ph)nRiM6&5&cYMRavKqbW>5yJ%6BL(tZ z4K;=Thnh+pd4^hiRK%y;Zbvq^<|CKY6(EsolXJ&KdEKCJ)lp6%9p)HT1VQ$u{ra7zy>Dk>i#^dgsM}f9g+Hy&0N*|ZP;!y;o$L;}Z z1eVpBuLam-ksVyp#j&NL!fC6e>dq-DnKIEb-~rDxYP6OWXtv<(n6X&BLmu}y@-;|rF7L=9K2U6&Y`)> zBX)Ha5T26L{U^CoYx}_dBCEp*sOc=emPr5(33X6tcbaZVA~eF=9|M-UTCME~Sp${T z3H=#xDh4}tLn%my1)&$LM+7Kptq{`NULXh!yu<;HUu<<`L8BJTO6bOFfi)joAsF0= z8K@N#*&H^>8>?_sh|=2Y%eGlFz~BC>3L{h0w7kOfTmXtXshhsu4G>QKeQv}|5b?DH6N!M2@*XWvm zNpWUGBESYP7NkR&&s4EUkqeznM`1nyC6-f42YW$!VSZ7b4fHR|h}D*>phxCrT$t%g zF|ck&G1RSM6+$5maDeELQN>%lRkTJtR%=OWG1NUMR9u6xODM=UvOj#MV6jjP+3U1( zwSstn;hBV4tmfGZ#L5j4sU)9Se5$=swKYWn7Exg3m_e6vCqy&~H3B=$vQclc=M6O)_n__;Ldon_8#a5dJf%RI6yVpXEK#_(e1-V%6 z6@Zc*b}O{Y$(6yAJ90fJM^~vd(5GJSXHd<_Vo8ogpl5P@y)=ohT51S!+#%*;a-J;{ zQ{G&4*t4>WCu?hpKoxVES&V`KAWx>?QK?=m#Iu;pUt>{SY)HyyZH+y_0=3I_m?liUT#;me<2g#wwY9ScVc*0B9vB1Ef1{>FVIXxOmnOzewDug6#_j}5Vlqe( z3!nn@N9#6ttc~k`ju0|etU0MBF>{2(Z^?)})IBRK8?pu!>~dsK;hqu{h;f{xs{%#?hHBWA}lrVHi>F?&=T01%K!aW;5`fO%Q5E3-iDKhG|B zv70vn%nmyYK!4Us&7Cd4gy@s6WYWZi?vbU!*g_-0ILqKcYDQstNj}=~9Pp^&{Z$(P zQ-mQEVjhaM^01z<3K&|TgBClWGZPY{M)moRJqqoy?DiVkZl zA>lN=Pp+}4wg}5kwe@);HR5KMh9*lY!7gz^ITWBm&m=UQHF32T;A#*Em~lBDI>U1+ zEvv6ui%Os{WPq3Hiii_fr!8HbO4KMYEoYQ7J91h!SbbEa1ICh)q9~}}c_1gyNfg*C zCMm*db);t}#6 zFk2LGH;PDA9_c_QSVel2EiO8-OI+u)%#7IRXlrI%Y(`qA*w_qfW^|@CBh%K|7MqaK zDJ>($)d1$-Ch9LJ$&bp+D~`>urK7u4)U(Q?vK)42iKxEn7}KFc6-GH=UPb0|QG*at zk?AbzjIS$p08W2~ zy*xEDuh3dE*#63lgD^5|SbjQe5iya`k+HFI>LItF!PYoAf29?e4a&+Rrdpl(5ojYa zPJoyUt=L&-ZIl!^t(mq6&}F$bKTot16C)ZbEULR!Y8(j(g<56ZHIojrNI@Y65f>RJ zp;chDm)fFq%|NtN6=$ij$u9g|R>O^wC~5WiTNOHsLF(X`&T??r@`CiJeC&RdMHLjL z!tMi6R5QTpko=7ukWHpS9kk~CSr`5al1oz3kgrQ*r?{IkY()^=;Ir-sKorWL!-o2#FQF= zQil@7MepFfp!Z^~)aztCbzNVL%Y_V>HwtBc>w?Crt0_`(jN&T4PN2e3q}j9ZQzz9y z8IRKHswxjkitTy-H1g=`h*U+QgevlP#--Os{mNb#C4^@b)NFf=1NkdT4H%}-LSv

)%0@*-MwJv``OSUhI)TjJtw`_X->t_rRTv)4 zwo#y=0%3!wv#GMDmtYa^vOmFy6B`*V6lyjoHu7oPj7l%!PmRs~pTd3gr^fdG3WnM< z3anT@U;!bX(NA;F}O&Dg1Rm{Tq`fDt&ut;2c3^VLhG&lMex*6 zQ&kc6SIDS3aHO><12W3h<$tCjHbMScr_D&RB`$dwHNkoRCj8wR2?#2+6o&Y}S)mKx zDaCc;{@<)qM-olRwAUTgSH}}g-4_Z$-L+GTZHSA}I^$r~$5NP?iOsC2lHzn1NBs-i zHEtB4!SnCbq|c+K+}jINrAfDXC| zfKN*yH#DK}bc&1>N148RGBRh_p|yZNOq$0D}efH~Bc;x)CPF6*ZX zTEK0iJh-?z3>s*iHsY!9l@vSU66%V^;LMXzm@n2MMKP?j0rqdjjy`sXxoM8`1l-Mq zi$FDB2Ap)E%ZavcZ}3miH~i@!WXD!Op~G|Q-{rT{E~1?wkG+5nWnw+A0Djz5kUoA6 z-FIK}m%{cq{2vMmwY*h*EO-JZJsXW0yu0W>5R-tT2JngDDijsl9o2hIe->+A5xh+x zpy?dLO(8416Wn>FBa-x7FeK|v38Un4$B8U-2wNF8=kslOq^-z;@Fxwwf!vs!Z9oGFVH6B~q`g>U_Ze z9mu4@JeFs5RKp_a9O#)!(MsgA;*d}w-PdX>TT`;Tuyl2CFA;SQ6Zjsbxgi9H3wYI) z+6(i57d#q8_fBzG$iv$S=Sg&@cY0vLEh<%dRXSC1oJ*_RHA}Ku&GSmS*}>^q9ww;@ z1h8DXec+Tr8GK%N+@v~c(%}vcB(Rvm)4i4%gXaxhONa!*Com0pb8E?qS;BCOtn`oc z$Zm9hfs?S?=~+6z)a3Qt(9`V&MUjI zy9BSZTKxWnBU(0G1yyH(JriD4S=Fk=v4pCyw*njVm+sEtohn_;u>)OWk^hwJE-9%m zqofGFqhvcbNRKFSdZxyjt7fQH!AMKrYE8TQeM3#jG)!ymXI0W=?YeiRtJ}gWPr-S$ zyqY}Js$|pa$glzApjCnn9!D8@I^5j76H`u~v1$_|=ex!v@lrNYE0fhc$so9Ds9UtE zJ7HB73Yt@y%OMWw^7NqQND7j(YTjqADr#nzZh`{jATkRL(xaMxT`jB#2SDljMHe_1 z7g8mWf|PW*sTgD}iO4P0;Ado2LWH*r`KPdfiSwZf&XW4#bgDzfg4}G1$4r#~`dDC- zP^Jc?!AH!-W2{VTpgV46+HJrG@vfm8qYZD-2J%;$CRCbI6e4;ptcMr^_df-=irlx< zlB=6!b#Od~i}KC2Wpe2nS~k?3M9m?x(2>;<-e$2$#=!+lhEtwe5)+$P{jHsEE1Se5 z#>FLcj*E-#+^KVPm-zUYgqQ?rQN8I2hbI84k=oR!ibV8>m{icx6^PXnw6&y zBas^EAy!=HG7SeyYWFW}qJ~55ah=r?{x|KFrjxDkMaHfy5BCaJ?r>mBW%Kr`Gh5Jb znvExnfzhf+Ftu0UG#Fl}d&*Ywg`XN8qju}2GEy^wx9iwWs24Na{Yi$DDj6u+blg?j z-Q2FaaBG*;u2y!#v|2h8+D-`#Rp@f7G=C8HFQDi8y%~sKjSli}^-71}(JIBwa5>QE1C3 zA{0w#fN(yi^R@6Zx2D164-Ev?*2nsueN=+wehpYDj~AGe0xLTOO1lQMR_^2=Me11C z?x#n$Aa$lhs(NFQfl7d0s9mL3JDf7F-`-tZuh4Grk9e{n{a(b4^}kgm z-VozAj%Oy)aSe$J-p^Nwn~=Y+A$Q9+x8?Wk_#|XJdVP5RW#b@4|D8A)bI|2A$qx@odch zEaF2A=^OEEoR6hd(lZe^;=?4wjqn)KomFlK&pqK%FblM<_Ke$*P^fR->TJ zi+cm`g!%|A9Sjjw;}XQo!hIEN6h`Dbv7=ziGaR=rEV5yThj}nl;y`yX+J)`oNLNuR&>rWX{7@dl#lcB!j|)Y7vXF5;S2-s2cAr^$VBhoYR<+FOn?@;nSrB)N0w?LCF`0Mx0& zNjpcZME8|*&Hn1gtDq6`f|s%gPKkDpLq-gc5(oa}jWM`rp-543P&Nd0{*JPGJD(s< z-l)gF!!z4Wa^e7LzO5_*py0IzFkg$w9vjie^a3+ zorpI>T!*QEkW@om_%|Q@XpKC?KfPQllw;p(IByRK3k#NWr&f-79R{V{Zv*2IF1FF@ z3DfEkhxkQGs;w;DW&>~)@BSf5W3<8YIBPEpBCeOE{JQ~hN&JTEWALsA%Cdi+I8}WP z@=?dF$fxxg8b0^Lcsx&FNrsLs+WkTjpLB*7K`tO^UZZHvJT_phNw!juSrVqxQJyqi zcF>+NQ^G75?eV^+MhF~V69oMaLltvwReMQKSXCV9qv1`iJ5XhbgO(7ipye{ayuBTF2x z8vz?KcOpbL`e4OV;;;?lA$je&cYaZEc~7fzupRp%II7)WJ94Qd4=F6Lp?>~x>Sqj2wqPYT7KR5 zqYD~we~BUAk6OOTy|C092~nfor1GbiEw9)L^9oGnv^HTx?zS$XOb0{!QkAy-FlLTL zZ(Fh?UcDoqvYy~~V~GY{u)N$+OVzw&gh9=gapslfXKr6|X8DpcJ64=o zyx`2TjiL;+TM#Gg zjq(NP&S4Bjn-;Wb7}D5?;p|}lVLhc{6EPBpmxLN6c#(F+LN{ze*$jQ01X$mIHoyiI ze*~}%!H>wJ^gm<~?-{2~b@zY}WjVq_2u~pFK+ym8;`yK!KaO7vLC7Q47l=SP$~x46 zAxc|>7=-Q!eG!Hs=*O`nZ%g9#V2ua08@CTptf*%#8VJL$9w*GXhSC2(;sS=o_%(%JJRDJkw@z_#W)JzmMf z)-*PO*matdH+Ybj^O8dTMGteso#TT^6KCm#ZLoN_JYJ4FjI!03PVQsD{ZQt5X2u`q zoAbjphx--~x$l(9+dOn4A$SP-4yP4VmW zMEo+nO_jJ>B_3KO&N1tMEU&|u?-*~30Hy@>DuTK{?)t#V*jgh(f7eyAs>r7_AMRcc z>jzN+$G>SmI|1PX0UogT)`ov@^^8s%JA$1rXEgdBA%^ulHa1{RG7 zQI?^fvMUpM4IR=uB@oQgf23T85gTwtjX2vt1yhSpojNrVrQH0W2l5;LO28wD$IocE zfia3DF9xCLX$O!-eyGP6A>P9fUxIij;yMhkR*%&4>pB;zaPh6PRr<*^{-G(stHBgB z8cF4F4N+bjr*ld{L^6Ze($I^DJYXx3<>D4Sncr}&o~Fu+qLL3)W3pm(FU=vm7>PH1 zknw7N>}N+kf?vj~@|TDz>70k^yGM|Ac)ztth%z6)Yfuqqls%R=!br6YQZOUS7MX41 z&btr9N;FYnFNV9gF@)<5$NWT|dVw@u%a5 zF;3i#LO@j_usn5j+7>$@#34`@Pe7oqpNP-}Aqk-?LN|o&2vCWHEwBfEEeOd7JrQ~# z^hTgvp6h1X1N$QkKp2QH2!SwWe2Dft6u-j|xUQsql4&Ef-;wwog)kan48m9h^5^jg z6A&gMq##T}NJW^8U}bRaAzSIz0Hll%S^~@@MZ|bILvR<&f%O8Gin!bo9v!-E2*Mjz zap1Nav6;gSv&p%o*fq3a+2gP&PhB6PWS|X>oAY785j?1}DB#+9q6ZqBNVqnR5PV46 zBa#>XrrEejl;J2W!hI_k_cbBPT-1|`KwE^yJEN$qXQ6klGxt%YF;Uj^D3WR|iTP22 ze4SD5HMCgt_ObLI)Vr6a9Sjp!@j~r|*+gy_%S+Voj7_-zJ9$YO;G)aIbUYLORb`;qOrE5t7vPz^O^>g~ zGx?Mre;Lo@S9<&-JhwvlC(J{r)}xcuEo!3 zznAe#n`Fo&`5T5`@~BAsHpK5O_$5CdgkRe3((p^3rH{QF&s=xqAe@J==K7K{fH28N zAifnM6d_DOa3D~vDn=+lAbw6oC_`Xf(-5X3%s`llFbjctP6fgo1j?9s2=fsZAS^_< z4PlWMzg_#i1HX47EJj$Oy}L{MU5ejj2zMjggRmT71;V`u_aWSm@BqR}ga;8GLRf|H zFv24Us}a^9Jc_Uu;W31D2#+H?fk2vi65%O?4G0?%HX&?AcpBjuge?fqB0Pt%6=55~ z^9U~>kjA$oyo9g=;bnxK2(KWp%~uh2AygvlMtBWj55nsRZy@YNcoShC0{Q0wgo6l& z5Z*%2=WZmPLlMjfoVOr^CJ4a@`kxOT_?~~nn=KPVl;;q3AiRO_A%gyQ2G3t>@t?F` zX|=d=0t7Q~o z!^Xz@^RR29td9;+Z0N%Tbf!Lj|FpO)Cv3HPSv$%GqO7j(a&Foo$Q8=}LpCUb0qCmi zMi}GHv=A}&z4#>#y^UYu*@yV$yr0m1YqJ|&rM*ZRufzqn&^6LS6kpVRD}IT)S7TSd zqouU1sL0PUx&ewyOlg0rZUocw7>rcGp@2eJVeNdmFRqH)a%~N;-$C3C!M>A zxM&x=RLlDq^0uilFCfILEs>_@U5C8aAzpp$O*ho)Z_)AUFNMo%_16U@d3g*wZ4!sk zjB6fR;n}K+g31R-_hXzMp28DWQO@EWd4Uz1K*Ajs&%fhc0DdE)%Oaw2RaqLI^>OsW zIFbx;UodLoke(i_#qmnu0kSJ65nP^;R&}rhPH%u!SioFmZ7ACadHKh344)ww%8t+` z!Btpqx3^YSkADy3YKpQESYc#FMD`c|4C$kLH*-RPxAKxo-9Gof_76|{$1$v*O8~=x z2-hR%@xFNOjc~QZb>p=mLZ6~uLweEU4m=Ms#OL9ew5O+6;JK3_-XG5m5w5n~<*h=5 zuJ-}!Kz7q}AihUKO@jV12R$alb0d8s!}7l~fM!a~0k z*M7V$LqFOePlV}Jax4YdD}nvQh9gis^y5*1tJ7*Z_vf@*riEEY+{-Ht%)qG!9nkoh zlg>Bz@PY$h9LmC%K5+I$9$Anlmkc};Op3R*O4AV9Wf@xyZT!dj{)_w?TafD{0VVW6 zpB-%jOByky^Do>5BrLSEVTz6`7l0=>Abf$K%dRi+d=}v=1pSY)>uZEM)df-C_Xv8s z=kR1TsTtft=ITXwcnA5aeI#vssq%=8qrJ!h$_sDr&Y+(VrtO$ho@yB~F17cN zoU(L(Aqxp=1RtrsKX zlz)ty{u5-JP}ctO8RUG33*&fvlsKbimK2Cfk>v>-o8vCvUN-tV8-aX&rclPP84UK5 zmp9!_e^UVh8(1+E4log3u&1OJNZ+pJCH3ZUp3SYLZTS}@kT&%ACB#V|dfbeG7y4>5MbI`d*9JTmfP55L2r74jwM04(BJHuu0OsCRqlz zzTz6*Lu94#6C5=p@t`GGc}*#Tb-Ot5ZY=-x_)LSILAz%O%MV=1E)ITCMtht>Q^O~gMB^`lKQVpUYa&WJOX3pza^%>* zaP3ZDYePUoZ%gzUdtSO^WL|7Uq|wJ!>mnqZIGpPg3bwdUU*GHCx*JFyo&`}o#5#m$ zif)REfHK-1XOsDP40&C<`&1k1!ZBm*jGyaDcpi4Ox?}o5ZKpxjYU>ks;o6iC(uj_S zJE2^YrgU4#KlMvWG%H@v<$bUQ_QSd)ph}y8SceD0o>w2XoWIhGzgzyVwE4eft`-8Y z;wmb3Q+p`Gl*zETR=~>XS$2)GMX7}C?Id&^&y=f+{oV3c*UJCYT-Qa)GjCf>(*Ht_ zQwTQ(f6(JTh;uEb$9d+8`lTMHT%=te*xmOBe?(0-y)ye z_K?5hd$zP+Xa25T zaZNscdz@Jny%xXAu6upwas0;J_)~6hPo?5y=kh(4p30gJ7L182?Wyb=l9j)EV^5{~ z-IGs`{j6t?#m|>r`%Keb!_J*Le)^h$z3zE$u+vfzR{LZ(rVL)&4$VpFgp$kJpS3zJBQF|=UvZ~jQ%XCH0W;oZaU^u2b#stdQx_3QWe#63T>Pw(FE%a+%4{w)Q*nQKnI zUiQ-y{mS>X`1$yAANLzH^T@cp=Nk5ZWMa?rFXr~?zqixKZy)!a(SPEaZCl&Tf40AK z-=k|{&YbDLH0glnliPCqh3kF2bythf;9XkfxTyP>UZ`=6+ zhxa#GblYcb2U>n_ne$c4@dKagKI~M?uw?@q4?aEq=8V?|j+y-M4})i39GJh|yWr7} zqXzZ*@wL;Zze*p}bjy#|PCEU-pjNAj9xu7}$e_EenYY;;UW0$Sc*|E_LlXx(eqH+c z>uCjp`(El&|K&f{4sJPZ-`nk9d4KTg2a@^}Z>l#WWA*8c`|eC0(*4a7$9F_d9TME- z-o^IcHVs+d^TaQ6TAvzn_ljdBYaeSi^!bIi;>PL+wZt;dFxSU`)qCY<43oSTH9js$LEGG9ChHjvUfLr z`SPe+8sC(0u;_-^ccx=^r-Xv93`oR^2gr!s5WLFZGOAJmLN?ULP@F z)$R%1)fdiA^!j^XY+!<<5cc z6dhDj#ti$!ckt!7l>F28_B+`}BzebSGt zJ=1&m?4)(^qG=|0TX-NXFjRek|_UxmQNn z*686)dY5O6KDje<+oYuT*vueSN#);(zHugBjF%M3pJ+zW>mjmca(C-wA|oF$nTx4iuQr)jTc zCO(|=_5GKA&Rp2b=giC<9kY%Nxc-^DE?Bc}zxLbl&rP{6>zlUgf4Hvxp{&MJ{l~n! z$dvv5iUS?}zlqPTm-I*H1Ge1ke!VWIJ<{gU?D6lta%n-FIyno|UnfYIQe$UIzMy|>)I@~q>u5EAU2fud5YtM%I7A$o{ zbh>Us*MgNz?6d7b1vQg<*LUEKDGWfGt~Q|f{rtHe!t>WNMY%!+zv^tdKa#k z=li^W+O)#%joMD>J#$Opp=SAS-M8v=;p~a~Dt7K^S#+e!%%h{O8CKMD(W+kGsPl_P zZrIcG#`Z52eYGO)p+e`mqJ}4w-8VdW{gly7EHP2{jGdA@?(RQ!yl~f)&)=W+NnGrn zDIYIs+3dB;zf2kN!|I0|mm?gx*r?H5+uzQ5pp$Hti7zL`GYEk{u54Cj&| zX6IX_6OKloN^s5@>F|E#!93^D;y1%yIQgj4dwBWA&&M8jE_`CleXY+06*t-U^NA@X zmf{xEiboxOr=<9i2j2dAPT!5i=~rHheKz>B;%>W~vri0dS`zfkuuqRYHlQRc|DGV* z6BQ+u{!@osKC-RkhXYMc{o(a>$??mBHqS~{OBdg~X6$t%MwUhm+4udKtlLXpwoc3W zboZ{(rZX3>z3IarOUJK@y0i31_|&3#mm)@brB1yu_LWsl(w0x%GV9wtE0gw3o%qO? zaXtE9o_g`MkkEbK#g=XVab)|Xr?Sc(kKV9#(ej7OvL+2Eil6aLnd#?=hmu?QmCx-| zJiGC!Zsjp+o{Sn)51)76cTd;V%bqA7JZ);=n{gkP7j3#D^Y=9kr!5{6QPDM^&$PnL ziIeZRbNVz}-t{vQIzKyYcHfSzyce9AHfh{1p}oJjX8K3>C&hnrWZ3j~y5Da8!fV0w zK1*(z_i3*k(+4G9dNwHc{Pa}cX^$LU(00bpx3%b#ynft_-$owEz1@G=jKmIm_HGP( zea4#?oZCm&yEx<5AC_fq8WuIP>;2bM=H8P&Gq8K;7T>84%pA11{ltAM4$s{3%gQc) zwDy{{szh1T?R4i^UdxpY15f17IyIrkfT0`K&ia?*XzR~MzCUZ(Qt#8PI@FuJU^k(J`@|;^GrJCw8%>rNfP9W|)WRYr^!7Nh#IsSAiV^lT>t7w_6P| zNxKz{7ctSX(Ve2>qT{0zqB}#i&bcv0Qjfsto?Gzgq z8y}kx+c`Edwo9kzPBEQgJ9X+5*D1bJLZ{B15<7K?i;jzli;e3P7Z(>Fmk`%EE-|i4 zd~|$Fd~AHD__+A^_=Nb*@rm(W5~6VnOl(4@gt&zGgoK3735f|^I!AYo=^Tq&V&Xc- zcTVWsxpQLYE{V~JF^RE>of6{`;}a7SJ0~V4cIkpHc0u#_?iecSf=peyczJo7%wDel z-I2O}LKAgeWli<{qs{!ImAgmtk2drFZsq?I0Q5g$4F4zw{Z9bU|Bm6RS^YnXOaEx) z|3fDEztzHqe!TP5-=Cd&WmT~;!@XYTP{qvIA^dOz^ghO*q-8wbt{KAvki zclYR5N4z(BZ|(z!u3vg^Z^PV#Z@+u;j&EOF`0B2XU2bXpjd|_X6;B=iF}BJ3zfAWV z`uXGY?l}3fyXvpym)16)0JPG%DeTJ*rmbyn|1wt_Oi`wjxPE9o~+p) zuJ8WzfJ>_{JkYM!v#X|BK0VU#t(DjQ_GZlNn0rsmd#~a9JxBNGG5q(|v(Mc;^A5)i zxtU+bhxg0udV12vyIOBP^1Y8@>lyp69s?du82w!O2Xk}$dfa^M)n(a3!cLV8z4^Ag z*Lt6QqSw^%=2IKSkNCXaQwaxs)*fxzbjrL3N5zCsy!BnLn{PYNYr^D@{Y_aOPfFLc(Ti@gYPd$`}~239#@tNXre1w}-nL&&dp`5IEA1W&d+PnB zEt`CG>B${w%IYgW419k76PuTAn)2{d+qzHQbIa7iuo)pMw>Cfg`n#cn6Sp-Ob?~N# zM$TUIu<7T1Ig|T5I-imp`BWVtmPr8}IvMY+TfxqjnEm+wF#V_q5vba$jkUltxXy`>p?r(=*#- zeOqzy@cawS#ywfy`ph*wzN~2X*^6`E8T)kSfI~4$Ll!ODvi;lV-u64UazW57m!E68 zVcW-FtZ%b$e~yxM`0y*AK6vc2yzWaLzTbC#e%YL#K6roj z)(!KzzWc;<|1t<=fyWdtr-_TYf)F5URn>_<;_ow#7O@2CgYeE;u>g_WP~ z|6st@C$8)Fb!EQ|Q5WvqGUoTYuAO}J#(%Z_;a@Ad{xIh73n}G|y3HNy|Kf4~?r)TS zH>d6M&6^)wTW}yFc;MpFk=bkW*Pi%z=D~nJW*$k2TJe-QC?Kfm$dt>2W~B6t?|Sjw z&z?AQ*lr9B?H_DD){`X?vL6GmSeoUkI~wbRKzHTu2jxCdUpY1(H!Ut00+hnGLs zeENjKo5xT1Ht)cXtL{%;b>#bwD}C0-INJHY5=$Bad^?l1iOV6e3d*}Qu>93CW4SIa%wx>qqmQ3^euEC}JAKn~% z_?ypGgzs%MuK(G0Upf8rs`HaOEt&Y?%NyU1n3D1B!*Q>7>{791>yZuj4K82uO4b{e zpQiR*^YNN_qkdoW{`qewx9Wd!L&wv%_i{Y{@aqekUFeeb(2MVNPTBO{;jlMK!du6@ znKbu?gA0n^zjJPG!t&VknL9J4{PM`VPcFPa#>>3wO3KsUyfXOth5jR+{QZx4=6SP! z`Y*XW&QkVY@aC~=2pBg;-kktXuU)E@%~-+zjb8cnswpt z9IC(WY{mJR&-_t2@5y~{J?C4|=&dkoUP`O(&#rstw>hc)>X?e$^7LzB_J04wo>g1A z`V=Qbe}3@y$*)Y^uIXrs^o?9-ezl%#_Si8-+Q+CpnYAhIq>~dXHNGvraZwn;qtpyeLvSRecP;&OP-7Ga49SCxt<%YANrv6k*{Lf?747p&fZUst@!qo@|#b_ ztRF8Xc5B$}PVW=%F6psz-Y1ia68qlSH0bHx$ND`xY-8K|N*DXK{Qh4{J{%M9(22Ik zH+S9NeBF^_PyTw_@iP^H?>|21%+70(;7O}||=;E~I>#d;69Yutl7-g@_%cD+u# z@kjex7JT-@xgO!W7kw6e@5o`J;<(+4=0sML+C*u-(BuU-bI;%!})rHkka=eJ@r1Iw>h+WBX4Y zZ}?H8?a%G}^85=w_dC|Kcyxo~cQ?=c^qbJl8F3T(w`zFr{u^FCygGMz+_$ePAJ%*J zz@@I|?62L{XM6Jc?|TfrHKqG+fw|sW+mHRC(XAaCKKA&w;E%s}?Noznep$NFvE%Cn zU!L!~e2w#?&Qlw+QA=>3Z?HBNf7 zMY83(BVP~67%&^7Y67^Jp@7ht0wf`S^ZvhZR*M<%6fn9QG1SA9j=|&M7 z5KvI;E<`{;U_lxz5CrVP?!v$h5D~irEJVdtR4lB=j{iEdGdk|8&+|TSec%88vt0Lm zu5<3wXJ!t&Gv_P{$-P|u^{e`*o+>9MBS}tdwxF)E>|2lcaRL6RFTY$|r19ltNYyg;!un^E z9~XZ;F}~+$$(~ck4@{4)b6OMjbyR4ikxEqghpmanR%KLtY2({Hl1HL6zQqtW4EDz7WM21q(uy*zw#_WQf9Kc1;SysBp6fRdD!(SvGY=NsQn9^h{~ z_kFmoa;|%iiTf=2>Aak`qwMXt{*xmP6xu&`aO(GL%<>UQ_4*}Vo*iont{eB=*=6;i zbx&^G@8)l)y+`MMOvtdH*RzHi?K}F!W8MkL{+O z+L8UGcy4s!w(X<06~3SDQoj8D-muaB=Wg7r8GP>Tj?o+MtH0f+=CE>7^!k=Q886O8 zFPXK&NvkAq)3Kf>FU*fQnCqK5G@~kP$(53IqZIcaANgo<;OH1p@UXmufK6f5#vQaA z2QNB)>}|{Sm5Z*lQM*6CD!u9W^t7E5FCISBWvZshx4_dwUv@gJ>5;RiMfZMo3AZY~ z?ta$jYeD-pHu>StcXpX~sIB$-!ZsNN6>g8e+72^slbE;LHh=gTutSYx&Kcsl7hs(jHL%;1kx$A}8joCXUevTOZ(%ZA0!{@sj+h0mE+gNts znTKum+C@v$I&aA7RM4r)b$!n3(3x^awQgMsACkUQWz5C!7kitve_N#&_-&WIe~{Da zc6xKk;{|?QrtLpp(WzUDsg;X9WrZEMd}pL{?x7*~^WzVX&saI;Mdtxs%B&Mp2KTnw zGjqy>E_37FYgWBi9MSUe7MtRFtGi{BJ$kFWn<$)ETeE#t(bj8^opK(ZN!sXVA9u2= zfAr=PvzE-#7Mz@{+Uj)KJ$;318702W^}Wg@70Trst}n@((Z9BV$V795iqDzi$LkOE z`q+26qlI?hc>ndw!oqvKu6yse<3rKh>*n)kcPJlg8*iEV;nur8duQ8^P3!Mb-|$$< z<4=-!zY+;JQUUlfS%^ z+nKMs^NE4n=^Cx9br(MGlE1T|;R>r|hst|gUfDx&^eW#4Rlxx-{M}<8c3Lw#$MM0b zaT97B`-r_($Hz_EUAuc(`K`hprO_JQ&UQY%-O6;W;--1$)~@x}SuHRd>ztN&!t%_J z@h5D=Q-%vJO>5O>#z)6-Z&!YF({RodOiVZx-8B6}qn)Qqzg=j#+{?b}-rTI}5~~$o z*X_>>IHCFWuE(3L58QTJE<0KM#&OK4t7`3C2QP2;;MuV<{DxrMrrXSdF-iHA!Rz1K z&8(Og+Ie;I=!%ZJ?$ksr+GBX9?Y=F!CynG5nP2PSdnhtwLB*zlk3OwDCw#8&lQt~lj@M?vH8Eh-7c%Rf{sS~q=myl-h#+ksQn?<>7ObZA!M?kRH~3e~r} zXq%`OmzK8b_&T+G#-WaJL+oE2+h(&a(C|go)19vkRXvI~dp2_GalgnWuy59c2A{Mes$RUfd--`AgCoVG5}z%O9y(#}u9&pa)NbYW{)}8On z+i)v*A?#6lr(y=dxM%{(XhrNt!6!nKPA+eyW`aP6TQo#)V3Czt9>~-Z`9B> zqE_~HM>GfBAJJ`e$(V(^r`!_`T$nMbbehlhO|zyAeVuB2?4?)3XA#>kHtOGeyXTAE ztC~B_41PpzHs^Lsg6OZC3oZR~|viBStIEjQGao~t^qT6-fe z_1xk8j+-upJ@V~ztjmx&I|GMZo7ehjPFpS2`c^Lol!x5-c&@yYb>XzCvyUfj+V*r$ zK?}c4jkZ<33d~yb%;09ko6qe#Iqay&s;-F7E2w;&>H1lB$h%;ll~d({)~9DTS<$xD z!e)QGyKu^-{c@dt_a(||rv%xrcIAEf+%w-mvgFj{j$?1nEIFUFbWGFUm0>G~q+VOn zFQtis_OT^dcJf~XJ?3;j`uyVNyt(g$=6__D-^gy+qgAr+S>1gLeVU~w%pEc7i1nmd za|8=x56&L1uhCjmlDy5~-JVRr$(Np?H?$`ov+VhN)U~8;%T4CV4>|Qj@AKP>j}5hE zg(=nU&G}<*p>f%p{52tSipK;tdRpribM9W9tG(i2lUdWA&zfDito<-2w{I;2cCPkt z+mO2ZP~N4FrC;|c54;m}WM9FY2RqhJOpn~>)cNYhtu5b;HkqNR^0pIczC=Ug${^)t z*DZGzOnJTZU`|d?9rc}^d~}@dl*VmUc(o?n=5&6yIOQ{Yr(E=N@-4}0_u=aX%hRgg zo^0t@-J-#osOj(aRXr=O`1+v6C@6n`x^LpHd24ohcn8N%ba9#UK3^r$^>al?j#lcK zqBf)Vc-X5Q`m)z2qqeGG|K-j`QNolNZf$l&#cUCkH_z4BcJNTZ!ghrgIUS5TY&^QQ zr%8hQ^~_2Q`|yT$`{|dQU$?r8mrlsxi>+>TojOmVon2?;s{4=+tuZa9OGUw9QvVRFP23uex709uV?F(J1;=xg z3*M)-XqUxuuUgeok7~u~D_Gv`W|s)z*^{qMvV8Si^DIBF(Q%(y9<*dvy#2aON0&&s z*$dIV&K}wgn;(^Oo0wxpw-5KZmGf50-Hs2^ifK7lRc9f?y~efP(LnL!sET<97_Paj zlUYs2GQ&gV^c5Vu{FdA3O}^T$ZVOWA>-bokT35&QNw>SJ+v(ER4Ft=3U)BH8q}Fg+ z*M?6hS1PI;ysvrXx!vdM>1%I-+EXhAY!b^2@}+|=AmwaV|N z+?pkA8c)G=zj%F3;H4Fuazor&~~w=ljiAo#6dqT)Zk;O6#a4R1ySMYrw`IQ1vb3jX zjbGh(Vut$5_xq2}i8QZ%)p(WK2>XUT9N%qeQ6~3fO5EeqGkjB)6~2G7NNL33Ch@AZ zUw2mZ>~PNGm7JgXLVYXiuFGbG7dZF~t0tE>WG<Q z?Kjtk=S{qQA$5Y-y;bMgi=3b2FMHGbOlVbM<+lLSv_qeoL?0g+y}$LO8HdLiTkE~f zsP}8p#lL=XpO)7uM_ze0Z{x1h_g+j+p7->`x%Sh#)RpV6+GqUid5<>MpN?E|T^ai9 zMyKbY>o%7?cHLrca^UD>)i>pqh8-=~*DvmpRDB#n(YkAW%9#$*V zjXF6b$^H75Ymas=*!UuM)yQXU%50WL?bWg{KTy4Kowy=W?XjEE5&O$Uc3ai1T$n)) z&sg0~<&T>1BExA)$Fo1)PQP|6!YaK-yFM$no!Y(Hv%kU2-6X!RRb`R#kS`C;=Ee=# zWhK}7Sz3tdu>nK2X6_UXbnTbbwb$hq(^6k1Utiw%__j8iLqelEpBPycJ6q>uooA!7 z4O{M&4{liz7?9TQlg06HujL(!grn{ssBrA0za(XGJDV`IC&Lz+jP&_nGWbYAc2~FZ zb8A`&!qsM{h9~`z9(Hf+Q{U;Q%z_`4jo2T#;B=!dl{%9bjD3`HcZ6U4q5WeE8y~*% ztliR;x{HqL744{Rr1|)Dm%7TMwNEo!Jr6uLM>{Z~-}#p-%1muSPOTs8TQ=|dw9S5! zNp@BVA6n}sH+Wn=+s&|A+3}Bl4~+^PqJn1iSsWa1W!1XB)18%-sfTYD$7H99w>nRb z%UCe^aPK$s?{^5Bqp|o?tm@Nt%@^*Pe{0&tA>|Xj%Pl)R)Ho`*H+J6kzQqntXFh$_ z?@rIR2Nn%!bL-i?<`Kmaw$3qc7W8X8Fn4m&^YMZUx6{k>MjqH)_cCI$)`nG?3jMD< zt(zKe81|f8n^SPctYe(|qwW22XMAnyoqyx{INhltW!Ia}*EVfCzWas-mzMQ^;oqrz z?&I?Z&)%)QWO!0Bext$Cn5AwO3l413n6UMR@K|)^U3tH6(feH%^ei-PkXz!PU6Z5t z&A)#1xJQ+t3nRh}!n7Yqf~Pdt-1TMF*aJh`O&@!@Yd~uK!Ud~hS1x(sso_3k-#z2r zMI=+KeEO(^;>G=cY#J15rzf|(>*+6V-<>$_oOm^H?6n|rL~Gg7j|*>H|MD=x#-#fS z((JPTvE8IWuP=A9wui4#_Bk~9$~!9`!}X&Fn41PG`bYKe`*2~usI7HV(~{krPn>ty z*>T0e&Dq9b+b;JIP*=oa+EZ{BTxkGXqFyxxENMAn!M z*Ds%c<#4O@wcU!pW%yLAYd84mzKWfbbWCek-I`LDd9uTv+0Dr|j~Ms3WBM;F?v!0! zw>!ssux|%}>TZ*#F<0$0FFvT9nbe^F0ky(eI`@woK5jq5xh8T`^Z6(8QVnjTuejf9 z=O?Yh^ZoYiu`L+V>&i)!5qB~!zy5kT*7l7_SySt!J?>5s5APRTcz=Px{W`ME@1fz= z>EoQoQdmt8m9*aD2#}@S)+c4 zag3Q>@QN8qI?W^H&uUD`IGnex#v^P%ZM1n~^UR6!9Io{W2srd*>rI8|^>)i+%1bmi zEUJ0cX7Jn-M|(CqaeQNKLBaT0aZ}!uwJcfZ^YMee>7EuAsi&*ccI70jk6Qdm(oU27O65n5-k&z)MC>2zv}eO}&&uo%J?-|)*}i<<$3Z*o=46Mw8kIDl z;Kr@|HZ77ulK0&I@O+`x$PWejPo{J@U#eXZ=grkI$sPiJ~R zUE>`OGQt{O*mh!ilg(GQOZwjnUcR7wng7kQ*PE|S3qBS1yp=HXvc=O$z51zjEFE>T zNnN1G?2NgO1qdhC!^b@1(t_};Ey?@b!nJaBheZ5{OpFDlkn6xc&51XCd{<-a?!@fNduMH~N z6tb1YwDq-QVw|;G``!4OZY0`qw zc7=IntrvF6O?!K{*G7+Vtt+Rmw>f>_##+6dMkD&>EK@!F`r@lEFRHp5uUP3D8{Yn5 z|Fk0|OXoJ(v2As2VAwLr{^{b+=HL20dAzpwd4*l#^;b^k z-AMkXH~Uq7m5=4iwaOLy>U#Cuf4#){$)ujK&$Pr@PL_5y^%rYDu8CcDB2sU<=8`^! z>B+07@6Gd#8bc%>diy>dT0Lv(j*aW*9KN1)viXSK)-9&Ki@&X}JL*l(yJZuG&Dq_4 z_d@?E)1ID5ONyN{yKGNkui@VPwrsZ+%sYP~!*fK`xw`$`oobf49{4O8wLsynxA`mm z*Prv{P2ZQ_dur-C;KGLIH=J7+9kxo@WcSb>45ml{JFDGric*WxG$Vq*Yl81$h zj!y{ak`vL#G4E~6dF`@&$2`~8goKlg?`?J-pHLkod9QH3$DVzp+Qa?M;vH6UegjvJ zPi(3m;xBjLY0l8CK6(j3t7eeFAFAbESg%PNba7FI*_k)qJKs%}f9hH>*ZqW#ob9)f zJys--6h@5oZfjN^wIY34?!}zCW5Wa*=Yt|0jwts#9;x(TRobWnJ^Vt(H*Y3(`Y?Xd ztuuWV#;-j+dYjLR<9ip)ELQbO8CsbCwPx__i64p%HCwHIZua}i5np>Wd;jEJpV#e! zw2$n5n=QK8v+=0c1N7zx#!t?0&qz3Yd0k!qtuKxC4eIRF|A0Z)B(Y9dyz%pu&k}E? z$Gx~YOzYv~w#Uw2nozLLexqNX(>EsFajIyp5V*E0&zX`mW`7O)hKX zgl+b9vK*UubN$HGQ!ms`yzuqu<3XDSYxdEdrnap^utQ$kbxo2Ck_)02yzF+PVd^`Z zF51rLolmyD_ib*ENpTw%UX70~A3y4o>h=AtA8u6bO*Dg!#e@#{sD80twegiM8%VF) z?>e+j8&r2Md97~VNR_OnPd{!9pS$DUpq}Tiu2TzI;k~wn!MM@Ojhn9X)4bSixw5ua z$D56U?$0!vX#Y6dP3T{w@+@%n(80x#(Z%;yx0|5fCup6?`_eb}d=&1RR7@Mbq-dyi z6OY$v6?NI0FJv|x>JvTMzWShxzR!Kj0g5Y3-)rX%eA{{Z=fyotHoF9zv@V_(@yh-7 z&DOqaS6>~y=+m}UV@SQ%e7!Z39w{g}c`xab`+Vpe8`EiBHmy{els@ywU2D72J+BI1 zuIzVu($nf)f7EPCiip~xxj9^a|K-<(GZ)y_ue-e>_SD99Hza4aYQEk7+U{aM73DAb z(+VbiyI%0Re(N8K^_@F?zHxPNv8hU%_j9Iw2y39Zr94o>clFxNom0wQHs5D>$uM?_ zPVcGbgKzbG{Oo9K@xmqPF4w;y{Pubh9 zfoSzCi$_)W1Krm=ZnvdK!!f(CmEJbdbak(_ukwOU+rGD$acZE+@$K50-RtYj+9thq zXj;Dge%dX)UQMDc*9XlsTe`bd4L<45ofw`!D?dDAbfNRusW(3?`SfJ$hLXo$=Nru^yW!d= zZNkUGMpfZ!+Kw|%vg>`bS@i5dHtrLGt@EbO$!}TJN8W#T;;V6m&0j{o-eWg$l0#14 zLd|;9a}ylfZ;eW}cU-jnZN!K_bjBxswpAZy+iI~-@EdaU)%=yyN9~-s=XF4~PU=XX z6z6uQ6Bjo2SRZd}S$#?EV$VEMf1p{60u#$PJ0ziPdf1ipGYrFBG$R-t>zDg9g}Z^=BJD-&+Zi}qEG-@9aO;Q4@Z zgDz{Y_~trxT4_I_`x3v!PnR}6uYURA`x}0(UiTPZbZ55wE}xUNGs;fst89HZOz+z6 z_QPhkGEE-QW{G(5icgC)A6{yirV^ojRD3s&_>Bn}G2^J^ zIkZ`5STXZma`aKsbI|oqnrUaeR(PlEu?;o4v!it45|z{AP6+GP+rDWv?5df~M&GmL z%Rc&r_HuuDYLAAkTlE~TO&!wpN3TC{dEce={m11F?s(=*(X?j)@l&1$=(?D$OB(QP zf1g*8gQhr-7}W5y)delrZ*fnZvJ*=NAG$hyaKG%LvEPpN{Nj3|$sca#UMD%N?mc8& z$-2>P_rz^)cf>`z<=H3Af@?1rRoG0ZT$?eZlW*pZtbvuG=QGoXAMW1JV@U7iro-G0 z?-kvt_G((V?C`D;C)R%a`r5w3)v4VMPS0x^I^kpY(yuSx&hNVI!j;lXf+4dPUvyvA zXW{Y~i-UOVkR)?w_0vwBIBFU<7Dk1i(e z*E^eZ$TIHk)Sa0W0m*u!$+*@dL?<&=(W z+?$rQE6$BmK6r0Hz|csGp_8{w9CGT&txnq(ZyV_8RI!%gFGT!i->_~8VchGPfvh=lmi;Lw=3 z#PF6_wg@XRV8>Es?HFX>K-XWu8YIY4f@LkR%n;UXz;@x-k2%qdi4%tHu+q|lQ$rGC zTejpjNx~u{ID{-{+5XPDb6lB?<)mwJbKFdZTgq^{uT*p7#o0FkZ^h$=;PgFeF5Cdz zg~#c7YW#3b8BW(9;fHI3)AzKw^wh!m`Ky4pm#f+ z5sodsgQYve#?TE(LZtf#$adS0GF?X+KD<0{%kcX${4O}Z4mZL1ZFmiw-v(!7_&FJV zNrqnl=eJn}I6ptSHvzxRPJr{;Oe|KIS4l` z1G|xsS$paEMfcLd3S?$Oj2HUj6 zB_wivdj@g&WipegmoZx}&fZrh+)F0hRwlfOOn4#kqwVyY@_w(^X4ufR{IDw>Hlzxs z+fHzG$Nov|He~D~mR#A2U~dBa=mWO3&=8yrUDuXh-|gVE54Z!x6?_4{0^g`7Jm@t_ z4v+_g0KJAO0E&PTpbV%0s(=_!1JnTxKoigcv;lfv=>iP^J%FBf20%l=kmW|KrtNM5 zGy+UnKiW4mPGi6vumCIp+Qw8{0oH&GK>LAeI~MlPTpLV5wfJq256&NlZm{KtJAu|yqRua$1e{-f`r{eD zybHj2V@Z1F_|0hI%1D3kp<{{TE8ynmNq=ynd2-{JJKH(?B`7n$Oe?_oZL&s&ub1I# z!TD{r3Y_0&%fR_$tovja7{vI;%yU4_kk_iu&2~U*?Pm~Ft zCKEnMCVUS#zrFU!@Ix~E064$B_JZ@|EKpj zYaXZfJu4n>47V+h&xV_ppNoGOZaRnI_?L!^_$D4)CYtrSo@3 zxc|=Foh!>H#5DvsPVeLVyssjR&jS!n$0cV^uX}t>_sML;3;zhtk58`&d`{=;d`|nA z&-cm1_ml~rEfel6lRn)Kl%L)S89rQwyUOsrGUdavZ9k<~ClgNhIp*g_=O=uARK|Xe z3=f4pzda*m_)r=96dCRAF1fBsGuPrSU4vx12|K4xRcM`$L@xe)POz+WEIqCl(>_)=e zHRC-Zdm{}m#O1DE-f;h&{^8P6zy;b4X>nWxPVcK6mjmZtj|e!uPjTU2Q5HI;IsOIQ zkHK|DJw; zKOnoeB*#1AW=8L49QT&t?G>c+3NE}H+!FS`S&MaeLP!MN)GQcV0VX80nUqiE zaV|Yyv^BrHqYQx&Tk3sWpqwUP7J2o3EP+>F>~CzxU$kb==sfYS}r~>2d86}3qJtP@6*HJ z{4qp1jo&RHE&_WP$6_UB>1x(aX3Xm0!OZ?K!Ppcan7j61r%bkIXbG_kx>-BAvarSX zJm~<@=9M7|oIj2w-~kAy;{u91GPyG6A)J36Qr-ntT=*1l`o9pzr-Ae9*d1el#%FG) z|JW11Uer1&EY0?xwdEGT{%7q&l5Dv(kLjAF^gN;Cab5#WX0OW!aMO8@^d|-B{sD9y zZMy4$bdl%8@3w<6o@xJbad&_#^Y|-pI!?LptKeEZPQT})|JQQibUsJtj~u6SL&|0Q zh6~R`7>46_`w*579QE*_r(R=}tufP_eG4KX+~F26uOWyEb81G4n9`T@=@!+`g>;SSGqWcrdd( zxLIp%ms)1^`bcbDE4_c4Sy}xqq^()cAn(qdgF6QV1@uC`-LS813UoR0rsr#W_D8X< zDd7oN_&XI|-{*)uVdISr(?f9r#Dt{d45gvc|G8-$dLjJpTw4>d4?r;D^+7y(?+umx zK@NVL4V@3s?{PSO0U4@;|6}{m4sYECi;zn=TP@hqvHXv=QIaq$OWv9q^V05&^PaZF zKibn16s0t4`&0T$kUq8VfW4DweP*Mqv}_!w-wQMW$7<^_QTW9XS3pTr=#TIU8T-bt zrSUJKeZrt0;XXp|zul50>CEEr+};X_+}NjkrVOKt#^VPN`qO4uGB$LHGwas7OMh$` z;V0cW!Y??;yI*iXmu|g#1$XxA(xF#JqPD|C9DfS# z2%ZMeHluC|F8l>+|IhB}!NpTSo6|PpxEVN|Q*pc%IPHIq>x0wp3_0!(PVc=Or{ja( zQ#sxd*GGD<;CMak9eBJ8+PDdiPmtk@zBm*!3}fMd5HkUl)V_m4GspmijtcJ3ehMuh1L5lm-VtsZ+5-N5 z3XXs?;%1>O;#$rL?|&Xg!)jOCW%efsCZRMtzz?1~Sr*k&dW4 zwWlo*pwJX9KSnx1(z?(#@k38J;^WSQQt80BsSm9+mDEj3NUZ}HN!yy1mbRD^Bc*N! zrp5dqsV$Y%huV4qO<~&siKAJxrQM9(St^uei z9TT*!w59J zBg+x(OyPhW9gqucN7{-GD47$|p>5X-E-Go?^kRGh;Bo_8ks=)xbR5u+8{mrg^vnqa z0sw2|Li;oTxzKYW03{6ssP+fk09Uje9eZ>X`U7r&D{`k}FaYoeXnwr_Drx?`;7g^{ z-xcS1cZGOJ+D2vq+D2vq+D2xAAVzkE?0}d~sE0jUuX9WaCm4B*k;fU?k&#u5v{pE- zkfd-4^M zXQT%sn=!H}Bi$M4#z-31jfv~Z#C2ujy5h=M8}5p0Bb79+E3S`J(zvd;Mp8-Rx-xOS z1r5w&aR;EXuV5JN3Mts(HvxBv9?ZQW2s=?YVQ(QPu#V7iD0i$Nijl)0(||bCCkFTL zXwyW4FsVq z(ZF8$zDTDp%0PJ^@IK&u!26&SiQtLgiQu%%AxN=-c_QlA0Xc`DBveK-GL{XAMm%@q zN@Xk~sgE1u7A=Ox<8jHS)MeG=)O6wd0{}}A9-4VMScVJ`?BcmA^iyG~fr|?Hy zf0W1{aYK=R46J=n79X^V5B$85mN)FYVdo7yZ-HH;MzCSmw(07_%VMnihekUNpVBLF zh~}^W?AsXzYj;Mv!8Q~v*ay87g4%jBp&@8tT1z+da%sF9Y^ii(q$?v`An6F9(vy+R z8R@~uW{jk9=?I~c#-$^KN*cG`<*>^Mm!~eTA-B0KfPU(7+vSPNXO}9lC-_`&SqS#n zr4FFBPh1|jtc2Zt_&#Ce30U3%qhWVkioi}F{1)sh;igu%T(-D8gWpY{#N{F5dIGt}taR0i&+hE~U^}SeTh1$GkYIqpsSqs}LmsLy+s$5nejMj4j;?ewQ{dX{O3(Hm_eOgy( z?Vd6&+D1pPuZI@EU@*wzQ#MlDAUC;*+avLt% z8c&e(XQsY8;68;OqOoWzeRes^gqI=KYH%964)JKqZvvP0D|K-_L*1O!CRjgVOR>qN z7WschtOYKoSxx(pg7z5IwC{=_sa+AR_q$B#)fwlbuOJ-fL^#ff2%L!#7`bi& zr3gBLsdRyC0eAw<0S}-V&=hcI+%7nCS^%CvbHD>=2GEZ?K+i0C4Rrx)0eAw<0S|$y zmPAVosA?r?scVI3m1qgHqO`Q((gHLA4L}U20?L3Qpa6&fc|Z;j0Pe`qo%y)oZ0?2M z0&E1Yv*;ZBF(b9QQg>`RmA!gWd3*|$gz>+ZLfyj$QaQR8mG`<(xw8kAgPB+>82j`D z>Z8Wkv}OF2BdCvEB$c({RGRjs^88FHZJ5xeG1MI$Po)y$Dlk&S zNO?xeF;W1z10y@%Ws*x9d}G@N-=ejIv^ST2Y;gaexu!ERjghI0Okrd)Ba;}J$jAgn zj$~v!BjXq;VPq^LV;C9D$PtW;Vq_#EBN#cHk>QLCV`L~JhcPk)lFlcbFxp!%(vy+R z8R@~uW{hmgNOwlMG18ThG_DPk2CW}WMXEhn-JI1PtZv5YrmS{nwHvElS?$7TX=>86 zq$x?$k)|R|Lz)5?-&WvcPCqUJUo&^iRHB$q8fG|BR}@_R@EtGx^usko>I#6%6)~Cv zo;WY5bY~>>mHK!v7W7KtYevspD%}`KZKQ5GyQBG1Nv}gxx4^8x8+rA@e1M+ioiX1E z!nek8*xAb;cXwy(B1|Ra?QvIk#=M5gP|U_KuQ8K;8soQvcK9ufewvxlkC{M8p2$-v zfYg=uVI=j@WqfoPA05Um=z@=G#+Wtc(FhvW?Rva@wh9{8R#&iDV=)`!<;i#a1wJ(id4+TQcO^J>WDW|b%xK!)1jHWJ)Tf)YroW`MC zs_BT6a;c_qX(%m&H10MwF6A^1WFt8@v0;3>PXu<(x#5It|P7LNXt6XqK+*8qmDGIBTegwdmVABBd&GC zrH(k)5vMxhSVtV{hr2FCqIRq+CMwN=TW6 zjFXVD5;8_YMoY*j3CWOjtk&t8wNs^F62}zKUkrEOwA#oBSk&svkiII?K z2^k?FQ4$gamD z8<5Keq{@I?G9VWX$OQv(-hiAlAe9EB!hoDLAZHB7X#;Y~fSfcSCk)7O19HrO95o#%jG$02I$OJW#qeimT$apo9rA9K<$T&4JR*j5NBcs*GC^eFyM$*+tni@$} zBPnVmS&byAkwi6;phiZjk$5%oN=#mg$qO;57nA2=@=Q#gipdi(c`PP%V)95#9*W5W zF{u@k`(koWOzw)w9Wl8rCbz`orkLCilj~wqBPQ3xn*()Mt zB2p?MyG5i#M2baZmx$~XksTtkO+>be$QBXVEFzmkWTS`_iO2>KSuY~%L}aaq6pF|i z5m_xFt3+glh%6V8Wg@avM3#uiVi8#+A`3-ifr!i(k$EEWNk~2l$p;~MFC_1TFv%E+ig8(o9I23W>XrxCx1? zkhlnmvyeCmiKCD>2#LLr*a?ZPkl4`2{RNTdGzEG1@;~LCCi4sv{YE=eZ~zYkjDDNwuA-_ldjeJvKl5nT+jnEO_p{&JsDR>G*p^L&)g+&TA3Jn$e zD^670ulP-|wNjc=iPBr8rpk%RTa{lZJFCR1Y*cxwVyik-b-C(oRa0@Wc$xU7*hFoR z+G4fqY7N!;1m!|Vn$rh9$iT=i9@SQC(Fndw9OfEojgSgiRFyt9OP&_4U~&!C^n+kn}y@X+C z;~Zgua5Y-`i13o|KH6I-(iK^X97SzJL88HE`!S-aq9y2o647x{HTvPH=(9+S=Pso6 zF~;{|tra?>XND+96vm=|mMCmeC`T_9DU>OkMqfQq_@tn!*ig|{v4vs)`YlQ^9ep=Z zae?A`#ZvTPrQ$8cdi11*lBtrjQfsBoN`sVQ(5v}M^Oe@2Z;vZgD?LOHD=6zJTPZhH z_Eqks9ERS`R-UcA3VnV+`JD1?^t?btQ^ichS*4vySCwFlf-x#nRhD2Z>{U6VavdY$ zvx->NSk+dwg=&Cme~gZJ)lAhH7$aL%52&8QNO^$`bv4DNVrOwXaaW9*IPo~~bc~%X z;{D=EjG*V@uVQsI6E#P*Hfli_O)+Yt)uv!fZB*NgnneG1k_qm#817Ke`F3)gLgLx?jRYG_*7tYf$FONb16~VH!aieHk9C5uuUvJ!F)| zWX3&;k=YviRJ$umW9*}kYvEXgkVI(ut1EXDeUH!i{9P@^H7Iad) z#-{%m`n^U9@>bI1=g<7iE5KvU?2!RI5;Gbb&3P&U%wa|`{`BYG6hSoo8C&4V+Kv=- z)a;40QeYLTnV>mFbE;;RV2S2Ngzwfo0eM+-4buV-*ci=}-mor#Rt%FCMyZyWmW!5` zR+^xj)=>OUNpl#hHBD=&)@H4JS|`CTXx-I%r6s4Wt$jq!SldCnwRUIiLE16eqqV1K zFV^0uy;u8;_I2&2+Ml(>I>tHkQI~(HX5XMQ5?jMxDJnXLPRXJk|NEBi1$6 zb6W3L0oOFl*r4pk0Hm4TABBYcQ_C zWV~9lkiK4nV*_+agX0Y@LqBNnwt<2k>HyHgjMYx8miqoIQv_LUGf@qBmSG zMK4#c6aLSb`zLX@6juCXvzf7Cr*OKGvFsROsK-f3E*kU)`XQfs;X7 zgDwU`3?v3)4W=0^HP~!G?f2m^k|PF}4DK7eF%UM?#S0JS4c!|0H0;rE7~Js<({K;L z17HnjG+f?rYr_K#&o#VFUy5K3Ixb%{6c}n6ni;wndKq>z9BLSEm}xk}aJk`D!vlur z3~w90FccVR8krfn7wcSLS2bDsFn$7lrIAi0uxDh)NE&mxUvt<%x+E!Z=Mdc~V$nyf_l43lLh zi$n{V9^YcJ-=xyymPx(IHxmu|V5`8iQHsF1(ZBa;g_LG8{X<*f-}`iGG_X;0qfw0} zH(J!Fs8Ly?(~W8xJ!$l*k*cYYsl90{(@v%XO`}amnNBucWLjieW_sGR#`KBlCsS23 zBQu(_y_u(3N3(urk!ERTxn>1sE6ui=9WXm*CJ@{(duH~X9#<7jZ zG@jacN#jk8%Nw6GlAxV%%jamnNK!fWL{)mW=>N-ZC+#k z#Qc-Fs)dn-y+tdFP8I_#Mp&d<p(^BYEf*RBB1lFu8PO3tE_9SUs->&R!)Y2@AUKbb{P^6ld5IW|zbdg+v&D0Qw%Mo)XVHo|16qmgZMpKcvh{_P z^mwU3+fI7W7UW`OF=LKnG<+;VyQ1|l4s7XtLVER76?IemQQ9R;uZP(t+NKDyk;h2E zOt|Qg^i$egx<54SKRhvx^rOYk)#<&P`;24a4ErBCJaDW{t z=g{X}6#FQSMi2ciC0T!(($!z&Txa`%O*L7dY}XL4rAS*`nzKMKl`X|Jj3zG4Q36}~ z`pRz(dPP16>yh8>X}q70F+YWI>HhP2KGOf|V~k**a+aWlo!_67mZtL8_ji6?$vE3m z1-|_5G}t z`bh7yKj$Z9EgSUB&2W znsbKWfn$Q+TSspJ9;X!O0hnPqaivLxg@=>BQy-_{PAN{gPSP3V&v|jF{!Yh=>90P& zKT=S-aoVLZ-Gq z!hUwqTK?5RuP*=k!Oak83S5i-+_rzkik*%*U3Pll^wvqiS>&c)8hoG&{+aDMBo;G*YZg^jiSUHZ5TcS&){b(xFjHHz_!%3tO2XW{>B-G9sm zeva{H=j*TYt-=3azyD-mwy{7^D!3~6fK_E$kp3i{%q7L7f;=Z`a*lFAanL*Jvf6?TDJJa| z2Pvj0%GP-PZ)0*P?N+>k?|M0=q4eE_dO0ikDupTKD6LjHqEt-oBkYZmuCl#yTje3j zW0f_8^OQ@JtCim=7ZVi~I~6|_8dtUizvkg8vi>;J~q{qjxV^?wp(f6*>xf*fz! zZnTf6rU&xD`LQzJ5grdv< zcs6)8crJJ@34;a*)>8ALd4vnbdjlwAHgq-#hdlsZ0A2u{3!O_MkRAY?4V_IQU0JL#7bT-;~0{VIa!TW|x zcmmP@NCMITNCMITNFw|Jf^Vw40r1)2v%z!0b4e07K$5@#62YdIoQd(0N&9R(#?5%> zZ0Kz03D6VJr_jhh8EpX&{72^v5PZYo4M693XCQBYWMpPTXOj%%0U&Q_nS06!yt3K+ zFzT3k^nzbL>+S=0UtkK$1vqZN-5-7ffmy8oP`E<@8Xg552`u2nq5g63OJHr1q2n^A zqWz|lRjl0_R%aq?9qZ0!?P>f?tY1F-k`Wgmli;2V>}11=Sv?DWrL3FAr3c;}^plF5w8mIs^lIZ93bzZ0TPm#=M6yTL+9f>&BJ+`2b~X{ zkMk)H=UX0hK6E~b0|%hgNisM9oe!OlafPuhpko~8V*ivd{{qR0lg-|Jq75%`USAM6RU$*-Idj{dHlbP)49xW)P>Gv z=zJv|?omJ%K<6rSp7P(WlbImf+YEhaCYS&ez?}t5@aFQR*N_;LgI=HL+zIPdd2wO? z3;p1VAoD+sM@aL!45;4~89&bE6#OYT>LzbK$?z1nI~^*6yeJUKJd8K!JFShPzZ= z;6*t<9zPD}Pv>R7^5?>77{4tjuLB;lkahFZ`e)rEQC2!m5?Ni!j-OPPb8}t%ni|9( zPk4vh>osh+a8A?tq?X`%r)}Ap-X*ZvMKw?gP@Da%J_>yj(8B6Wcox=cJFo-T3G4!j z0X#YDwHu)Axd$i%=v=iNz`Ao@`+)-h)@NbLaTxjta1=NO90#zToYzU9R6fW1D1dSi zl!>5B1ZBeYu>rqYf7^|83DPB~7eSo}>O`uMAFU59J1q;XE6tmhiI$gUa+TFJtiHkO zTdcmr>U*rNW%WZ=*RlEutDmvDp4GIyXuHrhq509W&~~Nn_pGL2pIA5b9|iYlU<}Yq znCab2nB(0;i2oCEGQE32_Xhef$now6-5(eL30D(X!pfeBzbOE{ov_5}z zcnUF-`!^4?6CTEt{_lWx`YF9WtUo^sv=g4<%*3Of(!4|1vWBucoYfJmj$(B*t7BOm z$Lf)+PGog5r*kmR$jKw=EFZ<{F{~cP>MT~%GXCnC!Igb7Yd4kE(^)-})w5YWht=~~ zy@1t=SiOYR%UHdF)vH*&hSh6Xy`I%Ytlq@xEv(+g>K&}!#p)7Pm$JHy)#a?-&+1BO z4~$uW9AW*AvHAq7PqF$8t1DPd+r|ZP0X%`51D!*v*sx~s2k`JOd%h$5D#AUW0X)W= z;|<`UVQILha6HbR@faI9&^dVaH^&>m!^YC^I&c6F5a)OU7(e52u8oJzfd|crY64?J?Sj)h4VqWp!g#Td=wbtF2jW%W8X8JF?oD)%2bc0{1W=6bJ*tf#E;| z5D7#9BYs?5}n&Pimi53+tYVI%9ukHgJtWHo2Qk4w|K z&*rHDtqO<%H9#HE05kzDKpW6up$pvr&;w|#^jU2H-H>(D{AgY@oW|wq{5wtU{#S== z8Ge_qtlgjS-=!hz&-KIabR)EtDPYF79nDiV4a%)^ZRM>M?BqKF*12}_SSbRl==}E& zBb3V+Fah{wZ47P>aP{KGAIJHcKL)um^6!+~0{Q-9d{vxt%D_^8Txowg|72~e*tR?P zA85l1yfpZAmUVOeD(la+4Od1koN_LV`u!)`jmv`@*OXiSvMo8w|E}ih#noB1zLax* zvToTpd^_26`Stzp*64qiI#;K^tN(W;{P%O>`uJCx>+e6yg=-sX`)79lPMB=jXj-y< zzv8lXvYd;<)q#fps|QWvUyb{B;&JCF)%GoBhiFul$r{;>or#KYjz{rM`NAK41(q2Fw9_z!~7%{-1EuvG!N_ za_1;#C(Aki|CIi#GI8NtIXKSGp9|ylaV9^Ub8~k8$Pda11pDQGv_xvX0cWzYwV0h$I4w`a>GYs+yiuYX5#Y5kez zr%&zp{C93yJ6Zm_c>jw3U!~0t=ezlNQGW{Fxy}z}+_G``oZsKnm`m?p(XttHX~_Ej z73bppO8;40YR?as<@_+t&E@es&iVhR^k0>UAI^94^XJOU>+?*0IOpcT{|&;W_dCrc{k!teaDH6Qp7Z~~ zOVBpIw&l|1^q<*rdHqQD*J<*@|IE#$$7$GR^6fc)s%8BJf?cT74q)3)eg6NjcOLLj zRDHicJDY?+2qbg}un?pxgpQz)u!P=w6#|4JCA3gQcazWr#2{5-1wdKQpA+K>Y94w?uIXW|>ySKkHBYvmAzD#`X8l$?czqnLf9gQ@(fd4^-xz z;{SIElYSG7`k0!ndV%-tW%~U^$jJJ9CW-vpI9o=3=L)~DfKD^NfPw!9JOl6RKU0vo zFaBlxGr4EaCAVdm@yxnq&n3Idj3XLoz7yM(J%!I@EyD~y6Mj9!XL~t@co|2lAn@_Q zOmC*WlbUt3#xebJqtS;`dK zRMX$ooczu3e?rYNOue@}?SJX_N%WmyInL`@O8z;OVfu?sBdrx{eav`PznpA|>x*xr zj(@Z@cg0`g z@vCbA{FVgpdnSONfBydN^lWMT$A_DJN|F8IdFV7l-zd1FWu*G8?IvA+X)EDY%o5CW z=65gV+%Vl{yxi%+Z1t2hf!lAvU*ah>FM!|N0DiLr_}#5`Gqa@$Tqj>1<~RwIKDLw!FWaznLz#pS^{a2tMrAHfVaqr|JA9Fez@^}h+0yvJcD8Sc^9?mq?2Bl|qhDUQ`&+S9-7tl@XE&10B<9Mj)y^PJQ_ zoz*REWld)dGi`YvO5TN%_nhKD-ft4*JtlbvsO`%t_Q}#JR^nUpGHsS}k39;<;5eLs zlkf?gg41vY&cZp6bN2g{tx!=@CR7ug6yNbYbkx{j~OqJznRYeJx$Xs;nqLN z!|E1W^0)e#<%wTby*VD5wl!{U>;&F@lr=F1fy5tz$~q~s7Qo%sNs;)n{ze7f#gz3@ zfSZC16_{uQz?~Pwy=m%1clG9uU2%~7&AiLt zX6EW^u~fvizLRObPbq7=NPWyTTg*7}uAv!6%DmIx!;JGgh5Ltfp-!^aOm|e)50QNv ziVi>%AQ5E!1X)MnQJ7BMSD~X}Cbk#ls1!REWIY9rRr8LKjj4jg+q}FgbwtoWRci=AR0_A1(llOwMWgm6HYqs!4}&}Y!4 z=u+Y*5I;eSRDIFDS|a{nOH|9_fbj14ch`oJAFyso3hOka*oNZw_Ux&KA;d(bK=EBGvQo94v>$@U$X4j&Rbv7)v-j2g_kGJRQ50^jr1x zO>B{>F*JmFP#bFU4*%RZ!s8-U6KD(#p&rzRnsM7nzgA+ zeGP)F`rA~S_BPeBy{4Sd6dFSVr~@?#yIc7E#HmW$iclWPKuIVHk%ZkXyZ~{+h?@uO z@JBmM{Tu!Tw^>W*&fy#Fsf^zgn@xSo_Yz+a|7JTGR~zlC@w?HEaK>*6xZC*wT@P+0eWZ%wx6o24D@FPvYmJLY^#y6K!AyHxCxS4>!X?t4Z#$4MZp?M*UCe_1~}8|3o|bpE1r_8mO$NA#r34 zj&jO3Cd-jWZn54UG0Ro_MU?X$Tg0~hm2<7gI^J9Tt+ttusQ6pm{&qRzoFVaie~0O} z!^U?}XRyy?eJ^Fk;CE7~pB(#mHM z`79u72FYgzS?g#di~@7LM)_=DuF+`LRn*!Zf$bxV)tvv#>sa&in&~#x^cOY#thtGw z)osQXfAf0V^b@xgiDTWDd$n!q0ON39TlqY~uj1(YX;ZuK+ul|_clqjdsGa#4PukT) z*65Rd7o?p9i7$Q1Z#reQnY6pPMy7n$5+t7Q??a9Odx*PRk8iG%DQnzFxapR@j9^W( zL8z~;vgV-gb5n=%zCM=zKf*c%N8uP82dSInV^cfY$yznCCXXO?Bx6y+tbg*k)BNly zX=SY}Yq-_6sWjT@b(RZz6@3L>gw^1Im9QL^z#>=(^IR%NW#6TKGN!F{Nuv#{Zqv4@k>un1^B)_g|K0TI^3T~8 z!QJBJ6u$UxraeNs6L2g*dVBO=>l1463$xf}y($=f6$6C%;=JC5{%qs3htuZ6v)azg z<6iV3OF2%99dFpJEPnp^KCFEEmvDzT{%P71_aV|Xfg#w5mbBxsCs^E*uqRvG{&`I! zY`P^p1v}N^o`pTz;`UFUlRJYnYhW#`gU4V7q{AY}Dcu`{ZGtyp^Bv+@(}%|Jy(L)v zOglH{8>N0?J%iiuE!>0~@Ht$A%Wx6S!RZz@mB@G*1OuQS^oAbL6*@tC=5%X_hd6i` z9)x;O3m$-~P!Y4!(h};0w48SKtzyhcoaA9ET(D5gdSh z@B!?E_u(CQ8#comZRA`#2+ZSqs59Tq`Jl>ngK1AU<%^oK`a z03?9y&q&VSGq~?JBkMfQI-Ew>qlS($RPMdXJ;hES_a`O&1S3rBNfuk&);~E%%DJm4 z>0-fc#7#AHrlGS9oonbkLl+pj(9m>4pD}caq00F8v2o;hYdYy=y5|& z8hXmmGlrfs^n#(6483gVRYR?P2hZH};YK0 zP{n4#W=MaUb5T0qb-^Jx3X_*G2B7p(#tv+SFQD@>&M`0uM!-S13Ae$voHe`Q99)8> zD_H*zbj2J+9zuWe?#5)#(l(2j<7Hngju-3{$&Xm3ON8rt8`0fr_RDtQkv?4gDZw}eT! z^)HFAq?q=4oyKF2H|%8WWW%0)8A=fr0(l@5%0phr2VsyO!l3{Zgb0X)LXbfEK`zwc9+@BZD^`QOG2Otb;b3T&mgOXr_s=YZ^_4HjMaKDMqJRVpH%V7yDf`u?2=E5vU z1sBYKX>Bw$3C6=1coc@hAb14&Ko95w9iR=gfM(F7O|WXf<2|+Xv13y;$get7fr=0f z4k!&Jp%@f~2ndIKkOzXm2EVtqsei-I@FRQ=x8Q5|56^axgc}E9RGpFKKYUUZFmdbgf}1)UWeDyIBfH5f7haLkW6@%NE`H~dSJr&3GUmuDay7QzCU2XkRI%!E{M!xJzAro&Vi3WH%XB*O$42T3p* zMlvpjLm~{q{SNn=`=eXYx7uZJUVaf?hBfditb_Hi0XD)W*bG}>E4%~m!FJdQAHZJt z5DvgWI0PTT$8Z>q!AUp`=ink-f=}TxT!Cxw8GHd>!VUNezJ{A{3%-T#;CtX#m>Cb9 z84vBzR_tFq8jC&*jamn(x`f4GKLFLBGVTiCfG8*qg`psX5kHuA`GKX#|A1dw*|9bC z4Yu$F_O(_vbr~*VpM_6|a}*BYw;$fbZx6Z)wzpEM9=wOW4YrWSM#9$-} zupIXy+)tr%aL*);3!Q=ARCE$L9>3A(2y_TKkh~wk?oD_P=!#!Qv>mjD7KFv&_b~Aq zkZ%M0YU5WMzZy^tcV)N_yBu-KqNNBgid~ra;e_SGzR{9 zx*K-F_LiD@2c3)mI#>&@z)SD~6eDZ``K=+|E7+^?^TH}o+^e82EF#XcunZPMI^hfP zUjXx9F3g6RkP4GwB8;P6E|`Hm9i0Y~LBhtP58Xzfn*X|3}e^=x}r# z>VQH}hPswQOF%K~!VrNyRL{ev1`|I4?D!2p`?C*ypeOd*oD+`25%>rWz&`i@cEbDI z8~zDy!?$n~Zoubo4KBk)I0tec`6L{J!*B@p!(P}8JK#Oo23z3G7I8dZ%UM7<4UIRn zrJ=12mFI9JUVB438rs>=u7-9uw5OrH4ee`ae?tctnqcT)LlX@hX6Oh*M;bcXP^pXm z_nxMoc@OGNDskn{ZRD3~=uAUr8#>p}d4?`9bfKZ?hCXBH5<{06y28+vhRSoz^4kn% z-F&t9>(>0tl9QS#D!k(Bmx_i~Hnggt)eU{X(3*xy`Z|VP&(H>jK4@rTLmxJ@siAR( zIt`6Cw56e~4Q*>^dqX=K+S$;qhITi!r=h(K?Q3X%LkAd|VCY~&6Ac|^=m>%$4Lxq?NkdN=ddARmhOV?_zaJyt1N|^SQEbtzl*!*1F}^&YtU^uH3f8pDD6iX8GBD|8$xA|EJI0 zt;&qI2F#n0jO{My-BWNCL|ikTStnDUFU;p8$bc8%MR*BbhF4$>tc6!$9lQqX;dR&m znXnPwfKBiwY=*aB3%m_mVH>;y@4|cVK5T~_uoHH{ZukK9z+Tt~AHse(00-d^d;}lE zVK@Ru;TRl;6L1nffm3iA&cInX2j}4eT!c&TDO`pta22k>b@&WEhcDnuxB*|m*KiZQ zfm>j<>%FLTT*<=%|2vrdm|JbdXVxEv`tG5Bh<$Vn_h$HfeZ8yi{EA*0lbg@-s@|2YLQRnvW@C z7N51Z@Y(z#pW{CmLO)Yx1Yw`c=WrMIE6BGW+~WIz+T6!a;(q+EtGK5yh4bcA?y1k9 zj8y9JjPL&ZCr{G%y>Jguu><*@<8ksBM1BW|zl%2c*HY?3_@I8=tEa59)TJQxtw}x& z9_PCs!jDmxd89i}y$(TXzBhTF@b4%mnQ+%U#wYtF?ejb3JxE)8&b@;|18G;Bg2A3a zHkQ#pAA@`+w|6Mtqfp);%9}|1rx^!|GM}XnDv+-AIKKbk`=|AG1@(H@nE`wexi zK)?gk;+2`rh{dLM{LS2(-muu|9FFky1wXHwxN_)M5 z{|L%?mhsnuGU_g3Owz`!=<5>8>4%QwK^@*?AK%9RGHGU!&)H7Yg*Iwj=4M2=?(2%9DFjHE4$^ zq^V9BC8_5P;#Xte=dn-CX{X|}zx4Ag;#4Kyg6w~yj3>(KK>lk8zfK>9kpC6(pUS=+ zrfn|6A@<`6>6&(>zo|nR(jKAS%Zd9dbLIuYdOpWHRP@s_@;XS_Uod~<`@3?}xkpP| z?ikNL)84)2Vbd1%8JCTyM;>^Ywm(k&KOx>s+NUh(-=toRslR+L*Pi_4JGtY;i)CMT zbpP`>*N5>a$GVBc7nQ$@#Hq%<9+<@UVf4czkO&K*2YvKD<;eGk61UJSpZ?4k5V!m_ zq#ynRy2IGqqMd((Dl4elD&{`nTd3FT^!4YRs3T?WXJ4*CXWD5MbqwjjbDa=N9oN8+ ze!lN~7c(A)(DCcx>7_eTwgXVr%xJ6?BsQ z0lBBR0G0qL(z z(7ZSIDq&C`+LS&&M&F)+5scUMaGJ6@&^DnU?KBXit?CZqn9Ce3&7AK7J!m)SdpSl& zK@X7cwB@)YbLJfFc+hCevSWSi`O0LTJ)%uFfQ*xIv~5T-`v_$waoh(PJLljBXhVE{ z%r*89?Z44z+v$vv1Lvy79D8EP6C%`Z z{C2@kr_)ROf!SaF%Dr*!r@R3hxc9LU#QmC%f80QB?x_=Z4ZI95LPm=~@y-OFZB~lw z=FMcT7wO!l})&SM{gRX&dU?*)E z{A!^0gAHopPDTHX?jp{1_yhZ0v=aVTo5zGSbk>5_5UE@(c=r;f!DN^S)H~2P%4rOhr2L%5 zQEtYNId%e#7yohMW*_~>i2vC5iu(J;MN7Sn{Kv%n=Ds;6Z6bXt?I3+cKS&wU=JT8LUSB-#^}!YVK81^L9?rsP_ykVCF*pJrgB{)`eNl8R`X6`= z4&gokAHrUk0YBmYHY#=b0CvF+cpv_&%k!^~w2`!r951ZL7i&BTdn>;3tcp&-Jg)P7 z9ml;)gZ1dE@Cv*LtHEQi5?u~UU=b{Y`7jq|8Kk1Z>-4P)6=uLRgUP5c5yrt7gOTWP z*hsp?28p-_!2sw7y$wzi-UIt2+7;~t?G4(XLQ8|@=u+avp-rF;G=_$>h40U%>fx>p zHQ|1!3Y82hpyi+pd`dn-DeU4<1R|jTgh40-g9d*P?>G1beuCTZE!>0~@Ht$A%Wx6S zf$W#;(=j*er6o7(Yws>FC_6K?I z)rwh;sCECPF0%j94oyMYLF$%MeJT)M395lyuRj0}K@*VoQ6;~A=zUNTsMudVhm>hJj0c@LyHF9CIFn8SpsFtjcFfkhV0BGjhEAH^}o+tKkKB30{GU>&*?c`Gu-;TuIVS5LBG8K zuR%GL68AFt609xfn_pT*?xUCIb3ZJCXW)7K4uUnG+}etZQ8O-fuKy&B8QE%!pP8Sj zX1se*Ise`*y?IWys+q>B{^eT3#P$#OcSoq%9U{~$mC7nOK1V{&>Wo54B{XbnnDwJ z7#`|y@9p21bdBIaXb26UKGcJ{PzP#5Ej{lD6@y(9YQO_hl6RbREYb^hM|aJt4ge-w&aWz*-mrgJ2-61bLq2IdmAL!K<(i zMgilj{Zp@S&eiR%?Vr-MOZ%TGdlL3!kmqBj>HeMD&p;ms7p#GoVHkCK1)T%q?)TMo zA@*v>faS1*I_!j9z*uh2SZ;4^W7D>_gVnZ%i){_F+SV{Ljp=8)a}z&MTr-`SpSaC< zRuz9aZi>n=Q8dt>nMUHsnkv>~tr;fI7z9cyX#&L$6xYlvkow2B#)(iRm|w-A7!-vf zP#6k9Bt$?#C;;J*AHpCXYuk+sDC?{{(;oo7gMcq0{L0PO1v^DZtO)#c$g&mR3m{W3S9=YbiTWd3W0v zzx?mj>B=dAS*{swQ!A+VvaI@we@=Ba)8t0|^DxWx*WBb~#*sY#PI_xy{L@FM!(3y0 z3?IQEI0y$|KYR%LU@z=}4`4U!f}OAfw!{1I9=r?hz&6+lZ^IUN3pT@>unFFPjgZ;i z+_yk#)>Xo+wl%K!n{KOZ`UOx|06#r{@#p(8{g`3ROWIHHPiNLCH|n1@P=5Y#tZs>a zw{ooU{cY?1_=k!6ZuPO2E%B}SS#5K!OMccot?2{Vl4qblYu;AdKW?Bfv;08inSTEH z+=JUiT#j|~Ikqo^XW(gAkjnRRun^{F9rOM3mAaYhYFbs|nRUCH=5%fqfB*7wa8v&$ zsqN~qTl_UG{ zPklEjY%lXWP}A-I9jN%H5QT=zz-$z<=KeRjKe}Q;{*{-=g zcl_OWW;s^P=~!ruW7^i^pua6~{QUyC@06D{536m>$4qb4m>`~)0r@?&I#3ttL49Zd z4WSV#LwYI^%+SWGpw{sd3)^p#T;sgqlwfY3P7Lf9+ajb5u zZH;5vW?fCURVD0h?O=`XZ%f)c`AN8WEfYxZR+^Ci!c$@63K+?YHoCeOXevuW}Snmpeo&z{LM zaq{e&JbNe4+Q~C?^30q(?mH4B#`pt`8_FLY8;v zm)rH+ZK_*4xi<6tCP#hNH|o+(a?9=K2Hxc%zO2*K4%)CLQcE-*a{K2bZX7iAOY^Xw zJGc3n@vifE>#9@MZL{`CZtb>%c+P`&JrD7IEbDjnk~Q{9@eb%!-jkH|7hl9(ZYa-s zj^lZeM9$B=KRSN^@2_>{9dq1ea5v+9OIcfZE$^v*+lylf?;;mTQn5?OyZd1Jmpo*> z$2L!M-O`_95w@(CmxnM}v$rep+*5fDg!ec@N$VOz{9)KohkVCV##`MfJN>Ww$aD8j z))s@aa0*VqQTP}R!iV~^%Q>~jGWO*bb;&9GZvJL31X4*W{{DGb?#he$NKcaT}KS2cJqbm%CGy{3Rc^zzmT~Mqi zdGw$jXlHaH$h+>}FlIl-?*|CSo`=>(Yr$*4`QQ0H`Y#A!PRRU-hC#q}0q^twH8i+x zamqU8T)VIjeVw)x%9iqBpPsG{{=J=paF2wm_&tk$1gGE^a1Z2M{2=dsgS@Z5)?Va1vk6;)S<6ZhD zAZruId-Wq=3d{ta;dItTYmQ+IL7j(v@7(Xh=Gji?FNBTZ7|k=DU?-&z1X zZ;D@IXao;B%TA^b3}l^v2Z=WiZGkQYo-cKFnBrUej+XB+VPx zb~o=JHD^qqoBK%o%EYe(72!U8eg~y+3|+{4CSE%BJXi&rx-joV;XU{acwW|d9W9f} zaTV4>1$YlW2U)A4D|!G9!Fk~MTFNR9<($>yIc^i@HWZsFVGidIUCTQA6P5rxcT3#T z&XL%iX8G3W8HSF4wJ-+?%;xxp${G$;U?j|fQ>3|!+U9U9!XAjq?*;G-uk#do5$d&| z{m^&nihn7}D(QS5zkTrhlfE?{WW5LZEdicMbgn~pLev7rJ@AYoX^J@yV!sRD>V8F? zC!XRwL_6}_BH@Ldi=Sp+Au?V33OS?DhKr~poQB8XmuEOXE|%~}=e#A%lh)kFLwVNG z`5E@^rSxNe+IX4x6(rvR&M|FxRu(?h{lcAZVDl`bGiWlsk!Cu~B zgJK}RN5J!zS=Ur@T~haRzIDs5)L35VIK>5x{-+gCZVb6bxvUBX*|6i_%iwo|cLmFK zXK~IYu{rJ#ZR_2eXHzSbsyQl5k(V!iE;neG9YQZ=~ICq!qWXhQ74omb4P?Q%NhXc>6LV z?J^^+xFxN)edWQzR-B~%U9FOu@%D5hZMu%Dz zKJ}#)x1^PDU#PU6uiTABT5(Gnar@MlR@{1kyvwxpG<*pgPZVoO@tChDo; z^=%?Pc3x$dt=M^WJFjljllUmzrkAm8_5@Oe*yHqVf(jv3oW2!1uWsihP6(;QU+K0@ zw|!|fY^igCUSgcS6K2WjmhGAU=@DHi;ah9b_xEw2y3)$So9Gwqi>h*^bgvwbQpFlpJAN>)S+CnRbZR zw~2ag@%mP5X*bypCe&tk>f2^f^`e?W6L<(3K_rAj803XK5DXO~YD7F;aC5`N%-FUyYipRzc>FUa5DdP zvBOS#PZg(c6ZDvI`Zk%Z%mLYoonN=}>$dczw5<3C={BwRvX-8q70A$nS8E?X zuf?p?-ddt9U99C>td&}%#iwhN7ihoE(`wAtZqC$>&D1W2YC)k|%_p=qk7+}vYOhbx zhE39rjny_LY4;~-2PSAIM`#gnSp*`C}tJFg)(@m?>Su5FD`@WNQyOY+u zllFE;ZFNV@TV7k#POINeE7D4<*HQ~@p{;XjgPUm;gSDrcYL7J4jyBQCKCHd_koIdM zEv$^Tppo`UL+yq7+UUC4?mAk&811|2TK?+ViAvhp1nuqGTCED&q8GH(^4e!*v}eWRCAAyHv^B-FYelrdVcPxqv`gt)+hDD9u+}I8C z(Mt~+ZyMuFW2|YA+!u$)J@v?gN85(!|H~%W_+L)nR!y)ubORrZdL?Z!C2d=a+BOum zy{&Jz7O}lo#KtyL-)<{n+f~@Mzp!lI*0&f>>pt^zXIf#~JB4j2x=+i(ww{G;Y?~Ce z)sD1Pjf06i_H3{%k6j(px5ppXj`Y)7chOGP(VAD$+EmfLX{K>a z@$+t0qq1Whh*Y}~*053W_T}0q`Ip-2MUA!vpB|!uPZv@l{3t<*({)wU=~`;og{ae- z8hpXFW5noTY=@5?!S>P7c9nU)fZBGR_o~hpQ5!DAkdohpBy3yJt!&>ZS__}8=WRRQ zEn-^lWLfVQsjXrzgq*HN%!VcHs_cb)%5foFmAg<%RlQJ)(#sMOonTt!6Qo8G&$gpN zLV3285~A5wPN>TEK0VZ?hgQ@>t0dHm>ZVr2j8@Oa+)zQM^QgBkRRW&ezf?r+x>Q{4 z=f&J0_AKR+ZAWV1r`e_#-o^IA!nN=hYoBhVVa+ToaapY%`zbxvp2FMC=Tnt06j4ns zMX8>bBuz^~+vzbiwSxVAHew~)=OSKWyN=}#B!^nmwnk)awlyQoG;Is#Q;|^xRFkL> zh+Ww&&8b2ycP4W%x`0U&OAogv6$nTCu2UzJRNf; zb8pPP%nxJsXC96@lDU<%+X>$uvmtryq>usCNpzm%o~{rl$}iaRm52pwE=7-F6F|TDN$4ttc5;HY(ddy>) z!wDN6Ga@r7W^Cs8m`3lu*~kH?Mryyn0}f4V;;#I5Hk?J9+^F3dS&*G z>66)!u#PdEGCRj~$?O`_Ewe?;;>@-&?K0cPbjZxB4#(t!qwL)Y_UsgUc9y-qKuO~$ zX(F|lOf9BSiy4^*!i%Z{;i2kqc$m5ro?q3VKPpC4dYWH74L`f!eE5Y0F<9sI?ZpM3 ze>k6CkO*(Hpk;X51x;AM;#62k$gg_jFQ|6q=PjZ9n%bU! z$Natd_su_${~)m!g)N@9H|)?n^2pz6zT|OWev`0#sz+Fu>PgD?!&sjr?7+NzVF!uR zGOVq>ZKZG9&FdagTy+l#RS)IOtKQFBK%HW-jD2|z&5co&Ln_YFRF%9{=f>o%HMdUQ zy7=wOdvI>Ny^!h}^1A9460CNHgut#@$3l+JIvKKKR^^Zivs#7RH>*uZrCF?Mr7Af( zsG*KZYK)_hddv|D-+SM3Z1L`O?DPKQ_?LH#<8^P6V}p0B;|=db$1d-q#Cg>5s&}Mg zop+SuHScK0dhbxj%idv*SG>a=YrG>IYrQqdYY_1UIWoM19WQu?I9~K7I$pxx(qND>;^WD?65Xt2mZ>t2$PAL#SH;@{1;J zv}2LCg5w$QeU8Q6ijF1T0*=-r;l3f zX{JIv%~b(UTUFQ7PBrpWSL37WtJ%?|)Y|9*YEv{vkLa&5-i-b_V{`P)jJ46PXRM3f zknu+Jjf~mRb2HMTuVgHWzM7E{{aVJF==B-XNs~gFl<2gK)aaQRv!dtV=gxR6`qPXV z(U&vEM^DU{5S^ScDSArAFr$e|%6Pi`v#S@CU%9%nJs;P2P1OVTZmNderdp|Lv`%$TQ8ieVQX^FX zHJ;WUuX?&Cr~&SYYOs5xYTzEF9&(RSE!>0E=dK|t(vzsFrVLXRSXQ_y?N)%F?F@k) z?yl+)cQ-Z2-B#6iw^NPX9aOyAsrI>=s|b3nN=i!=?QW&2c=9T)Iw9Dds7ks=sG9C1 z^_4qWO>oDlAa}ef!OtSqaCcNU+}+i9cbFPN8Dl78jA~2Yj$;%|re6zFkCL8I>S6jd zM0HSY=+m)`ib?cmA<8R3e?H`?z|FD>_5$iYK1o!xPg0facGXptNa?0ZrA$yIQzojY zl&X|aTTP$^rc%;sTL@hBY_NUn$+UgvnW}zdG(F>a-gd<^K@Fw`qp87l&m{GPXR?}0 z4P^8+@QhYYMqo=%lKO~I*q0iHQ^T#aZ$-xLK}KL5Pe;n=K^c9haUZqFlb}{JN`pN; z)Z?C>YL2J3`hhry7@>VUUDY7=XcT*Njq#pOwN?jc?b_^BeNTJUjJuBUk4@2P)Nle8cVZ~R<|%B?uZq|okWZzmqk5ip-9VeKrp+_m;k;qT8yC#O z?e_a%F8avMQTdC93HIaQ^k2fd;8 zMYb^eMipkikI#a&R6+Y36=@G+%oI>bv|W-am@-}!PDxh9QxcRtB~j%`8KJ^bIuh1V z6-eo#3Z-;c#ZvgpnG&x;QfR-FV3ka0vKr^E5cGYTGw60&N7adz7)wiZcHhSsxSxEB zscE$JH1)XqygK9lO`Uabv3=uSq}?LcG52`Yo0*Zo%t)l|PB3=!sS)ak`%(2RGv!NW zN)u*ED>p|S#&(o@nEISq;z;ShQK}npyQv|Rc--Ab&2T5EkC{>7%#xOj^`q`Csuwe8 zpt~jUTB^^ODP^hmVP?}8%%+E#8y_*7^1C@&F-J<#Z;6U0T{xzVSCOscGU%(cwn4w7wGX

O;$5KQ`7?X=?~XQ)qxrMlWU?{ z#vJ+0)ssB5n-Md`4}4qE|AQ>)SXceCV;;7uf9f zS$rxw$X%L-e1xjOIBw1!-=_xJil{+l^TTk@;Ibn;Hm=`~apk^0_$=FN!2@kWJqfm9 zo+oT8IlhH3<{xwSu?_L`wI$MOvuUw;)MX*ZH1n*`o8w;s$G?&I#nXy4Q^u;Eo?)aN zrphzUD{u^q5Tahj>rQ! z);-EGa29E5q;yg}I0p9ev>>go^<&)JsOFg09KW)jmZ}`{Bbv6Tl+sLT9;eF3yvRqp z6mv(ZNsNsj=-20Mq423^we2#!@(sOmi`G9y&%8;`Y^Dc7)dBSZz3>S=w2t1{!%=+( zd;JSN^eQ#v+j8QJP*s`L-?@_1kFI1j!rfc_Ob=aUyu4>zQygV%oMdbaclS__QLEjI zk9X;zD(>p)Csm(gV<~1s0o9;v2t4TNQ?{R{Z`uByhCx@E9kY~{FkhU4Xz-pbl#cMUBjrKVOZIH?j+7JCuz59wCgo>$^DtS>i$tZkaAMppYn;SmU3R5Bu;ydUn6-KqfW{P>Y{tE zy25d(07n$L)+)?#t2|?`I)0C`x3%S{;pV8}j#q^^K9yt7s&UL}=XR)aw3qB(aE8ca z&yTR@pL&O>LaK@u3Z*?&wKAS```hgKY5W}=Z6BcK9jWU56d$3$9g5~DO!aaN{2AG_M9y43pw$B$hurwXO62V5;xX*bV3F+zr_a*o=n zm7|mz=qR8@I6`0~*WMp-?R|l3?m=9059ZU#quy4IHs02bwp?#_;hMZF*W`V<7SG4E zcpa|A8*we(*c@DFaUR1kpW0)e###L_uG5ck9e>4JG)Sdww2$Qs zKb|xEMEY@(ce{Ox_X<~3&)Z7z=3@cQ_95^yqxS}9`(HWJf5@5s0R4K_`@HQu^Ya4p z^9J+tEBbpIFa73IcH2fzAzKkoVOvp8J6k7Dds}Db=qt?8oy_6mp5N4Yw`R*^l=`mP z4lr^Ls2$AZW1jQs93%U-XN|h)-KAb8&eyIn>KDe&_pW613;pTlXqr!TS7+!a7xVmk z`tVa%qWXfq*}>fX(A8QUa&=HY(_hCp_f1rVZHrZ-pf;*uw$DeS_oOWld6Q_#fe){GC`lc@ZSP%cw z^hX)`?mqgiBK;ztq4Lu=;q*-rZ=|EBw^2|(`Xz{d38r1|qhAVfeh!7>sfB~=X{Cbl zr_~ASnzqOOQR;sC$EgSGPp2NVXQpl@&Sv}i)GhYcIbUu_-EKd`x$~*iwf2Rn8||;9 zZm{vz9C4P|=cF#P&rMxne=^l;pO60+^v{>{&#ctp99M_irg%o!rgC;WOaGmtpNi8@ zUFfH-oF!VR)0}~BF-Lx5{Qss-(^s!^%-ukr7W3RvHR;2no*z|v&(G=)_jTf2SO4Zn z^&j`=DldJ!-E&&~&K1}x_Z9V$XAj3=-?93A&tX-AK0e|(soLTHD@U&1-1}50b2yaw z{U6sfHO;+7%}m{;UZfsIBg?&COKNR1)h7}cDyaN9Ljb>~XGJ9GM`Yow~h zm3$-jXw{UlAlCzznbUcg(_gxVsUO`vRSM(ef~$|3%9)5KHi*+nb>l46jblL{=KNV# zd)0*Vfs-+FoOvCpTB;wI_aC|1su-@?ALRJ7lX)M)yx;HY$B#Hal27ho#)3=jP=BZ< zwsE$NHt}QSXIm|8toE|@nO4r;-~OE3^;DnOiv_g^njXYD+(9>jss?i>B>1u5SA&lS z=M8BcGArcekn17$+)UC_Xl-o6ehX%4qF&@NVmkL)Gdqc^RaHD_#T*NCH3tXw##*k)s^%AkiTyD zgz)v@H^a*pXkTDv9*O5JAX|PvvVg5%TtRoiodsp5h1L~1Q%L;(?q5*h>?tp>FXiz|@bl+aL<^rRT)0TrBH82rkL!P0;D1`+zh?_9 zDsrUAw^{SGT2ZNGm!fG!-zYja&mP@%vS`s_zPi^b+`ZWU{QE!E0^?}qjm7@={{8nH zWA|RG?~2tbKCbx2;@=goRbpI;jU`T$C{i*|E_Ji{4k>vh}C5>_ZziI2{BXkH)0IWifM+3CT00f zD>b)Nuh3|gbMx7qLVqt+Aj)S)aXn`J zD`EKi{03X{w8j$q&pq=G@wux+{ke~S#xQIAXr5tNc_(IBf5yw%@~`n3o-51EsP0B?L%6IDnruO?G>H zR7%v-QL9L^Hfl@MzO1zQay56d=OwOFQJ11_X62$3@7xCX%ej@zv3%Y591sb6=UdWtx}iR%S?41<>J>o90V*l_UW zs#I$i`w6Yddjzy5%?M~so+F?&Y0bEzZAkklv?Z^RAb#!Gk5SML_b6ybJw{pTE^#^# zJ_b4vJ_b4vKE^LjN6Jrvj+CDS9bGeNYu{LrJi3s6Jai%bc<4g<@z52&3DA{#O@OY1 zO@OY1O@MCL6QLXSM3B7Xue*!>g6@==4BaU+8M-q*lA$|gCqsAIAsKp-H}{5Zy7&oUETbj)41=~X5<1dH zNf4fB8RHsRIQrk<|XK)&RhB()3TLQ$n z=Ad(!d$XB)v(Y)|9OlSu=Fn_(4myW%JDawjjm|;mFkWXfUT33o&^e5&*|ucpj-3o$ z72_4U!9+0Y)rxu#_p7@ZCK|_hOMrM6$1#Xh$Eg2dI0XB(j)Wyaq}q#nx7M4sn`&u4 zYhHJ<{nwrNC*0<`B-S-O#81{kv-+9#0sIBiZE8;bW_TdAuE}QF{_(QyBfnGgnLzov zVt*^PUwy1;%>1NX1Nocb5@u}+aeoOn4E$|tUb(gX`*y3Qf2rW_{TFv*->_y9!&$a}R7M zuRX9I{4+a--!bC&`~P=#7Y%aRiU#@WTsSTJK7|I1NeT{0GXuQyWZ7;~WBaiX&lxjuCfD^Z$By zG1|WV-)#Q~(#1doeH&xx-=g@{v*dZ2urp8$>O*m80A|~j2ufibq%ddF&@@|FXkrQf z0{csN0NO(Z;yR%Mwo}|`Gi^?&jNKBhjaD)RqlYc`v< zDUe24APv^(#9|9Sbo`Ode^cfK?4&Ai#S8^B1s5qE?R5X?EsqKBGr~EMIW`7lK=I;(pev%EP(-Tg7RTt&v zxlHLQ8C6#Ds%*~pRebiRozc;k+IzE~YvSjd@JKr9cS|GjdZR^$-Cx3EY}OqReQD`H zKq_DmU>IOH;6T730JH4>1>6+#uH9?zAFvnufHge!z1Zkx|BC3hO0NWv&doCKMSs7s zXuRw<#rxMC zuKW1sH^RqzfbZ<@{i6#y1R-fy%%x?+0U^ZyXyGnX`~IVj^i_FRUjhdI4z%euz^(o5 zD^l$;^naaLc`SalJS_EfZ>z;TarA%9Tye7+i(O&OMDa%Ou9y64tQK|+$NOqO6v-CB z|CvNFF~y~;{|)~gVtN(e%KxeB#uN*4DEb|jSK~gE*WG0pcb6eP+i2|dTi;yie-pkK z@C<5(*2()nf;pGjz>a0PVPN8<;9gWjuz9ZqmNo3_VnG@d(ss^Q}_GZ*L zRb?gDzx=#j{WNTx6G3iW1lm~VvO8SJN0# zTyeVe;8p1>NYb)>T_J-U0wY7J@`=jo)zlOcKy}7pcMWFWdz7nbK-Lds+ zh<&6sgwyC(%sqDid@Bhr1odL?Z@i(spH+W z>z;h1?`+UD9}Y?`5?$%3?|X#j;cX`y;WSmIdK6B%s@F_2(I7y;5#D6 z;`>R9t)<{K07}7Y0F;8)04T+Gn;HP6U7L_^f_wwy>j1|9j<%M876g>BEtVi4kkQaz1Jr=m4yXaIo%2V7uK|Vwh5`lw(yg_S=>XJ1rUOt5-3~x4eChzqWE=6_y^YAX zAm4)TuWhn)a(@RxKN;Ww#93!`t+6Zz%z};O0Q`m?zBk!rITAjv0N}S%Q9cdjQ&Bz@ z`3B@0EVF^H1k46+C14KlRe+=LUCY&gqad>ya1?x64QK>(a9eBfy*BYZ(UaG7|9*-e zI&}bl*J;R4LwwxuGYVh_SO7nR_55#W8mJAgLCxF2vg;7-5}z)j#?54Z;SRe;L@TLD`D7Xr=)?;OC{z)uI90yqh<8n7IIU%zIp zfM(#0fCj(Vxb7$-vAei#7P|;K7eQw;a(s`{zZo(Q z3`IOI0}Q^zj``9`m1}|1hpPn;~4+Z0A3XUn{Q<<;M{iH~8O^K2_e|`T4`PS~ddqRXrE(BU{hG&WnKO z_mRDQ)xJHkV;kN*zAh6j#)R*Q{l0WYNc-A;u1EQCn5mwC2``iQWD`y4?5q56CSqN; z8+A;B>|xNG0B8UnH0ibhZ#R{f17Be(hx@%6yyH#$Q-GgpDxVJg3{yE=|KDA{624pm zxE65zufVSp;KPZ44ST~5XAO6LcuQ!?f!F%(7t@=82 zf7t)E+uR?Ym+Wqv)w<|0#LeKcl@g+E=5`{N9nUxBeV)%hWWoF-_)GGM_H< z8kx_O`H?c8BlDwWK2PTJWnL$9vfUu@MKW(R@rZ9ut>CrhPUAkc0dJFdC-6>*uLQnQ z;>QC&UgBmudyALDZnzB>VoqSDZ;K#j=J$kOFZFJe`OPxFRpz(J{0^DlCG(v!zencx z$@~GC|3T(d?;j=psLUTX@xuLj7QAO=9G?gNyu@Du{*uIB0se}_UkCoW#KUdvi}LSa zU)lKr_}>Bl0DNhp6aS~MveNl)$s;|3_4i)*pK7D>?>|-lzN&X`+wp6dKO@EB-|d%* z7%0{Tfe|M?B->b`*@2z&`JNz`}I~KbC zxS07)f!$8nwJ(888}l6u-65AUU-dlZ`?Qt$o>|L$4QDam6APK|l{L&a6gHnchWSz# z!v<{r2%TN17j{H}G(_bl2)@%b6` z6(bfOAI*G83lLYtX;~-QiP-#rIv+s$u7xjgka@b9`Bny*Z#Zl{4E{0j10S&aDiGJ} zVCQbc@|N?M?@Y-34ErCRg7(!jUl8_gf_@Wxc>=Je5%r;Mt>8Zoy=ONt-#75Nq?Y;G z(Wjpw&X1s;pV6)Z5ThF}Ld<`|e47!UU4Tm<`{*L(TY>r(qP`BupO10CQ0H&Z2P2?Y zaUAoV)q%J{|7-Z$ig=8K?C0=x1o~|X;++aR+X45X-p4UU9zq{eJDz~OZ_wuB;rmUf zV*-5bL|i^V8~!y*#Om&Oh%4f?1N0`;^Bau6a@0|<5n~cDo{YXu+k$?mf(`g^JKFer z(BFm5>9F%!HGDzLC!!xOMq8;bcERV8k%TCFmy)+IS4=TRI)`h{HgUrTlMTBG zXn&B#6Y8pj{cFJg5Pj%`{r6yh725VJV)HKGX|&@#=uNCaf5V5t(0vYmZ-MOhxMsG4 zcjQ*)yA1ub8MdB6-Jjt4c?x6Ddpz@PL@Xa{Lz@xr+3SEK7Gp6k$HR{(!0!z~0`P)7eStXt2pD}aeA|X=AN;%E z*G=f_kE`J$>U;ui`2a8-aoPqSowJy)5#WQ5*8m#kGG7beLX3w-^vSt^F9B5;3-19o zpbcx$4=d3A48(bOE94R91KWTj)*Du0T%rvXfXRSrz%0N#z&U_(0Y1ch4d6kaW`XrE{3C7O?Or#1wu0 zJo@%kKoiF6jeu8BS0!TO0#KY504P=k3vtiIH982_{0zV>#Etr%?$McmS%6ize(AnM z*UamP<5M!0LzannUcG|(3K7$r0W?mILu{R$Xd__oa@_X;GPAdYm$7{SV~fVuQvixH-GBcEpm?7Rpm-M{-ff8Y z#T%G!`9|CeHi_8Nczqo}eGr2_pngcYM9hmG_GrMv9s&F@u8p@E5>e0WYn6 zqvTI(-!6F_@Bx7A{I@6d3=}ChKd*-4O8&pS{d?&DgZm|u zD?5GVkz&)Aj^gi*ZX&yfj{&^9xm#l)5pR2``g+UBmwvos-_?^13%kQ+VYk`(+oqoC z>Zc#v4YJYS`jp?KW2(Ey_9%ICZo%7l7It&5Z8VR!a6Hu3jl5qh?D{_Y$HMTo5|V{o z-KVU%jxfB-R5s&^tkW#)x2F=(!nU7kVOJ2Y^N3D~;%<(;87ICe##HPrb@j$H@w)I^ zE{Oj&iUIIT^jJ{cyHTHR(tA z^8gnB&Ig=xs+Da7oCjEcD)hU;ZBUtPsQ32Fc_i6hXsG{e)OnW83Wy}SS~ zZ~c*rB$LR_2qdzVfh2YsU=83T!0~{U0s8$Bf}Yy`p5!aJNcPP%^PHk^#Sgy^P&r&i z$@Y|o+vrIy5*_a#BM!}IcPF3?&jaZNPM#mGj5hu`u)x6 z-39`RJ=rnqnagI}ETeGcgX+UbxsvTIS9Zelp6+Fb<5{{0;0JivlE4ByPt*RwJpe2A z7*+!=2GD-Pvw%wrNbcAG?LE{28UQN*9@Z4t3EpDh=kRi4U^DWo08hjA4an)4-NP0K zzCmuobNbtmKNNX6pbfABFrZL8$E$q}dY*S+Z^H?oy}}9r?R6XlXa}4KIN1aqwkWV{ z8J?v}7p z82Ai$CiW#p0bT*V2>It@vHt^bVXtC>1P`kVJPiC50PUrI1@N#1fnSh2=b{bR#~6-$ z%{c5|Ag8^b2La~*JZyd-Fpu(M0xigw08RkB3wRH3ND=m+kdMZm$lT01o*XpobhNc$|nJ~0agH-L8E=LW`Kvy z47`v0JM8;Z0MZW?duJ}>A3$y+#`Hx1hP|Mz;NRGVy`_M$M>GudV*oV(533DSK<|6t zYXLK`|3-Uo9##`L7x_>?N)!BmObPPsfWHE+IhJH*1YSp;b{O*w!+w*8O%K!}uSB^G z`DFmwhq?#gVbzF#6`(RuiG5ZNn-+Kg`7?mUfPvVnngF1^Kiaoy0;~kA1$fxhKmqbY zS}=wH`G<==t;c|S*b#w$f!ESX_!QW$2s{eB7}p)`cVz-LL4FGI69Fp#Hv&9ta-i%u z$|nW3^1M7Sx=rkhRU`MXvOpK|i#T2iyMe&!c3e+@8zwN{Z;}5VIl-h3eBJ?|eYCZJ zCjprgnJ=r8?34s1@Z2Au&lo(+7w{~nd?IX40306p5wb3y*lY8!!$3O}Fdk4G7_tI= zAc6MbXn$)R@^a*x03KEps9Y&NTX`D!`MN|HWoGt1GMk^Dc}%4qj%^~mv0HGPjEx1NN( zPe1|69(D+9<^<+&{NTXqlZkdv;3DMT185)C!w!VZ0f7Y^&kii(`RKq)(Afdpwg&rs zr5F>)?>LI&M?rohKm%m)>)U<`?xE-5dWPHv;OhX}0JqJ+bw@eiKEOu+4;vBq5c%L! zabE@82p9pl5AZR7_JV7WKM8mm@CLxchNG@w0B_*HGTgTz^A%v=TH*~2H1K>#U>Gf-E0;C|2^16(fmRB1nT=6d3%1+GW_2SDaI8211VONGwB zz*E5Q1^k`U1_WL_SM0xfSPJ;bfeX(^TLFn1NGBNSGt(YGGyQX3<8L)U8@uCCY0rxN$>Wcz6 z1NU5tJ@IMq7x~Ws$7T5J2QUzDD8R!U`a043F221N#n?Mv;G28uU+ev7)t5NA_)pS2 z@h1TvJDELzZw%&GmRfGKd}ztCR$I@t?y|mR9b&7pt+(B4``$L*KE-~#eTV%^d!D1s zvEA{kBf(kfJjQvc^DXC)s990xL_HZ5?V9X5!}X}k5nUR6YV^a=7WYK=$?gZ-Kf5Qy zoD_3^%ug|g$DSB_Z|sk;hsGTrcTe08afimQicg8niJcHz8`}|kLF|pOPse_S<6p4@ z(Lhv*`K$+haRyyk{rVwd5#jt497zB&MA(Kjw{hew>chmynvp1+wq0t7xdRq z=ONCC=(SpBgR=vD*Wx_Mc`kbJCg=UmUFgTpoj*I{qlQG~L`{gQM4v8-S{`*edUjjX zjZycZe_xCGYt;AX<#boJs~CMf+tuh=i5}nVy4rOY`u%0sN3MUO_Xk94(S;ZTwb2dH z9T*80L~oD24dda3==Y<)!l+1e4|9*f*r;;Pcei4MoaesGeKW?%v+lRuUtqMv#|(+d ziJ1^n8FNfb3r5blF_*^Ngz>X0=B=2|F^b}12ge>9dwA@$*mNebJ1FkZxFh0@j$49}wl40Hxa%?Ao{D=t?o*7qnE1^21LF^kKO+9<_$3&D zXT@I}e{KABL~LjL44eUM;lN{E;-KDd8`w4Q{DGGbyk+2n1JmNt;?fhJ8)$g{h6y*Q z0{aZ|>4`rMbf;#d9*}C`Bbe$LpK4r%6{$bjiDq!)9dlBNLN@Awr6;zf&PA(ENnOWd zNqBmqz{6{dR2D7U^Ru$K-LX_}!Jb(2@D`;fF66DtO6tqTM#x>6nifa>wIlVR)L+>1 zT$A+POZ_tSKC3k?DeWhDWe!ivP4lIdr&XuTO>0a`i(8SlHf_6OW7?HC)3GD%A&%`z zdo3+1W|#G^X>>369<=2SclvgR$MGb;i)5r{rx&MJr2m88g{sr%r?;k`4E}ilIxb7U zIsNwZKc>H!{(k!1o@4QvK^^-l-IgK77oABeIYZp>giim@o8xc~%^>+iM}EfM9f6F! zJNoiEfyYBhUErZ>qb6fPMr+2ZjI($f6mN6JRT;O2P!y+oetQPp@5Ql$>-;f;EWens z%lZM}>kO9Z$sC&bzMSop>>-&GGu3LUDsz5jYvy*xae#Qod+BR3H)LL(8FX!Tyz9Ot zGXZl9SyY-LpAhw6Cfyr1CD0vsYszz(uVzjj@NuTN;?onqa|adL3T>ATx?#}W zl0qY+KgUAczwYszjT)aEMDy9Ryl$%hZEoo=IDZFl4W`(n4Hm80?ih`75nwVv^rfh6 z_Tcp?jezwjD+gCbZ+DzIIDXI;$Xo;XJ=cC5*b9T-$C-|=klThJ5`Ymya)+pB8S%c5 zj}#FPJXjAAS57tLw>uUL$%5`T8DK6fE%ZFSXchivj!F8sP54)hb z!toZ$+Z~^Co$ZdFd1>Lm_@QLA(=lXde*#Hyqn<>sMl<4)jm z>m`5d(CdfZGxVvUFN0be{n5~GhT6TN9Vy;=*9b4!$aUs>vtj~XT9wZ5F7&o}ySxXa zoX>fedvEqW;N1lslHBfi%lo3awQch-|o0> z#N8uiLjTbb6~kWEEvCg?5WPO-qY>YXuw{{*-8;m{SM`ZIfjB2;jp)(dW<3TCks7|kiIW5%Bdr7vRymq zPCOaD!A6Z7H?nNxb%O+F#>n|2>)6tfCyrb{^8As0{7zII?x0GijtQMm&gGE1dE}?; zY}*6SeF~-PM}Ew9j(iNde}nd)LH%Ilj$w2d{_Pnio>9LZ$)8k5#*RuG1jm@pQZlawmn!(n#|-&Kb2WjIJzhciadLN##55>p@1zlH9te zp6@oqALD$cM=Nv;ZH7${j^u*5wJ~O)e=$VA8gO29Z^u!NwEs})=sUKz1b7}a|UE54~ErNB$|Lf?< z115)frZ^(smQaCpwDP?!bz1__>)_L*0Tly2%KVU68#1 zHo}j9lpuOdcH~|bREj`veBy;ZmZa(3E2iL$r`yTZLa+@&OY%ICRGbX}-R-L7LK|Dx+JuIT9O z=!)p3=(D1)jb0M9Gy2u&zen5LBisS^lBl`v6Wm+f_qxxDe#8BvJ2i%M!admC^0=6A zjagxoX<=ov{A}=6a6g1E$M@jVCB4`X_HNg$^c1y zfRq8LX{pJnsnL2$O-)NnOHH-(m{L*)WDLx(_Lx%A2Bh^-dqQk#I#On8My6v>nkO?O zGc6`2!Hx{=K(g4Z@-NJXu%a*mVQ!yFyLX5PNe{_}p{|10-`wHoS<)VsGLXz9DjSJ&WDy@h z8Vrd!_cBo%?HLw)9BqfosTTSqIwQkE!;IkoJ3N6XQ=~SEUIxbqUMNON7Hb+A76U?A z>rWo3Ml&t6s_vys9<8XT?wo#6LWO#4VyKu=G9nk+R>U6n8zgg6BH^jYjAdq~v)$_R zWM`^V{E62L7^91-heRU8N=;S5$UQvqdmnC$nIee7hg1*T?!lH$w(FQ9qUZVzHkyLq za5xS5J;PE}Js#<+-R^X9)Zs8HgYC>rwA7rKr?1+>ThMDbdvf1o#9<0F&=B`}QWtdH zh~1uvHrN?+rl&_%AyI@tCc;FCF>~XA_dGVB18Fm#J04zNh9r1AoUN+tvr7%{bQOVk z!pfmVS9tyXD?43Ihs&i*xsaSLml`yN30ybk>EAStNd_u?G@GO=kT_trc0LWBDVXX zm!9zIso(X;OUu+CB;hf!jkaexMBJq$8X#N6NIOYBVx1-; z=4+bi%qbJ=5zOr35(&9#)QH8zpXp2)kTSp+I6XRVZVlBdLZ{Ar@X%69v`vwz1N)J% za-$?&MTs|qNFj?YQ**;mlXHC72zM}*;gWz)M&m_%gg9pb<%1tr4j*L{3G&!vk=qbV zTqcGskl?Xcui}A#63(W@X(~N8=!SS?^q0$ z$$Cu!$xL;`hLKVRK-EM;U2eB33#(wG^~QaLiY`+xP+3ay%z+LLDZ?BzWT(rh@jf9E zDx~#%x&t~!`-Hw|ICd3MF^rrZ>Vc3|g{+0UYw91lJ`k-iJb?+SoS%sbR5~O4rQ4f% z1;h0Xb<_118hgayRN&z5@EE|9&>=(Q56@I{$s1ye0FeT37}ZWJ)2&dtL&D;@ z-1J~Z56VcU=Qm#97I5NBoM4;!S5Le)l*lGdD2jWX>A7EBQgk^{5|51nr;<#nW>Hk0 zs0+kurH6Jpl)REtWt&XAjO&=}*)psdv}>goc?m5QZ0rZWB*q&!i~>?8ZgL z%ghv2J46|6sF{=MC6Vi$L`YOr6bFMCDvTSHg@zwa6gnOtE0x6ln_yW$pg?gE3%@y9 zArA%kmxiqmq|7u>G7ZlK<_=31D-o~&Brmu!ANy%gMwRMM(D+$Dkw_Ye=!vVFPKrTO zBjLg*5K&ZYGC^9ZOnG=&_-DG=sWmGJi-_lNj%?i#Z^fmvDK=81V5f@0f%q^LZjUHH9!h8)yt>uf(1#@vG+p12X#BXB0`D0_)c;^W3=87zV;H^vzz7^U@8nZho#76R*@q(=jd5x)cizC`jLt$*GdNh!ZAd zxfBtN*`Cc#oyD&{FY@vU~fggzw=h{I?8m&%{XmAz7UazzkLj_rTIu60m?rCnmN7mAjb>zl@fA9Prl&JGtq6~C#!OE)5yO;-7x{~%ZXPlGI;WBf2Hl5%>vjZUaDic@ICTR8(Zx}p z{tm_o(p5+!d);$IEY=PnjQ)WAqvm5@V8 z+`x?G4!0wW!Wk81ehTQby6PK>NwP0;iNKuHe^_|P$ELS7G+sf)RcUKU13Q#ZvC zx~|o86va~dLu+MHQBMr|ZBF{9s@3gt0xT-NnwEqaUKNH2CGHoUgeVX}m!=E{!RNBd zVObRNoI@wcm7S@Rsar%60#q5bPYD>eP2$)cG#dDLAR%Y6oNSbInz~Iu7NNoX&Xt6J z5YnH8q=Ixx=ksB0YQ6Bqhz9u}_cM8q^Ls8QiW7HAO_+$Fuy3?M+?tiQv>m2T7^(DO zklt*%Rmi~eaPX%)@&mV8-CNX&e%Fg6FAZ`(_%k5sPziTt!wt-5AgeBsK~O`OyfZaP zn54VAY8NIIy#*YR(rUj#SC&J_eFU8qe|h#nI&2~Dg5fXMcV{RXHJL&iD8 zQxY^TTU99|Q?ygaw@k5V>7-BR^wDh$G@~U-6_xPr5gW!7NYmPijF7jy{~$-ZDGD#+!70U>DPPH2yOL4W^%}W^)QtJ)mbNiO$cacvJ9n+dwE{dH{j&2+9** zlF`05&D$g-Hvox^Z2`(8O>W3ZzOjx|Y+eIdk`ih`M<$SnEifg3JtIL-XKq291V{39 zx-2U)7AMisJTM|65)~N_rTj#K2TDFVP+(Mr*xff!juBU~IjWnKw+gzjIgN8eh8Jze znUhqNGUh=+QVzAY6XOW?6l2hEE9s_BJ%~Uv5Uo&@yg>93t?#w-iwh@lk*QS6GJ95A5R@I+$LQT(gU7Tk>urxN?;qJoKFOZ2!`}b7gSUS9?U{g zNd3Gw#fd*e>Sd@9uJ4yg6FOhN{y<^;8ZohmX)2N~7ceCjyB+2lOHd5!ZE=g6prsH>tr zh#KL#!LiVFn(JYgHM%IeD*C+WOQYkguSci5Yur~mEYX>G(|2O@{OHxu8>4TH{u4i* z-*5SsJ=mwkbZ@_h@-2(PD|@35kI4DYEZ#B%zm{^CWvXSarJ2WXtK|knkp#&OF}6(GfwqHe zhuNmu=Avg7*_PW*NB>-IyT$e(dg&$Ghj^>}ZQB>NUu+5Xq4q=U6YW*#x25(I?dQl} zSGfm$_>}#1`={th3%>h3%#q`m;HY#QgI+z+aSq-izrk^@<7vkm=;0q6G5EOm0Oxq; z6z5Us?N!dRoEPimYn;D#KJI)4{r{cQ6_pk>GHPs8Io>pHi0X(sE$Tvyh})z781*8? z#n(}GSBh(dE8i7x)nJSW`wLxdu2WnaFwArc9NopN9SP#O^%)& z-H35CKe{#gWQ?ZEqHm6V0AuQ===-C0VPp-q9&9}vF#WIvAY6et=`@4 zJ{2SGF2`ezKRakdAMU=u{VluFooT(NyjrX7bPF3^{&|)_f7Xf z{E9PVK85{r;F$u>aj%ZH#8BOdIGc~NA$e6;JG}hYWz159Ph(3gOHkJ&X`v>j4!=Gl zq)x!-y~%P5xM}>#KPl#V*qrE2vu2>YUP}E2FHMvD1oyRJUJ$j>y))*a2y~+cszsD` z_xyCs>oJwAk{w|AG-iqAhnSey)Yw6>2gM#1J2m!OHaC_IvYu&ej$IjhdhEfLO|jHg z#0I(fxIL_F<|+J8=+RM)hy_Emsqd%5E=k@S`$yC*j_I7D^dsjl_M>K{sqFTYE4lvV zlSA4-Iz8DX`UkOJ>FpA7b3(=!#o4HL0=Koq@}sGBhxZZR#qmolia(C)OpIU4*0S;OCs>|wi!oRgKfjx`*7%d-#qV;^FP*7tS^0Hd z{AKY9?Gwf4W_t&;bFD35dNg{zWhL$b*264^ zvXc_T$U8fM<`X(-3|^Fw%17e2>^g_w{}#U-Cirt4I}(HqQC8FoFph<+)tL$H1go+! z(S3E?$If%5{Sx=&=%Y+y_YVo#^7=5_Gw@vrc%wi{yaoJQRwc*!JjZ(`&7&QkOAifQ z1DoTh4p|i#9EE+KaGl93K|PDt7ww_aK+o?S8pnL@LY^y4b6ysKGt)$y><)T5Ol8tN z)nl|{V;IkDoAC9rKOTtx7{65%%H8InWsBFl=34ae3I6&wTSPv6&`XTwSAsurp!tyC83|((OZh=wpJ1s?r1Q;|YaG`) zCc2mOG0M$8HMr*|E;ae7ddV#FE$cZZ>MEY#UTLxts;x>B`tZvD^G%6n+La+CGpE0}n)B%HDvs_H#R=ydk_I+TpAkZ}j1rBSYSWr-iREy%Bk@=Z z9ZPT*e+cnctETA~C3HL?b5RPPWr=Ex#XZBGNu+tVXWeJq)ws5l-zjm=xJfnzE6WLJ z%|!MpA2mJe3rRi|hLg%hi?#EN@$Wu?)3Nw9dDlWZi1rVSV2Ep4Dm_Zo_Yg z+18kjwIQ+i{KK zamR=F71&5NhB_lr&9od{!VhewUNXz{7w-tszR-Csetq^W(7pzwM3I$(s3}o%qmGZd zBU!2Wy6}Tq37Gob|jqOUn-W^xE9&)`Ka_p`>{oNc_lq)()FL4BKin*fk z_8R{b!|k>?ZB8d`@rvXWANlZo#)zl>@I^hL)tXCAs+~SWaWl6&iF~nI33XfPWPVK~ z#0AsqBO!EE=!>u9ddy|b+z0){gioe2^o^2^DI{cLcMl5Woe=`wCW=#!PWQnu6s0eH z$oIg88T6B7x7q_~{-=KovzrKQ{Nu$VcbwfG=Ztf^O_j&>Cz<38R2Zc*K0Sn$#Qs?7 zO-7n_NcUhdk)v^O>YB1zyXp5)tKqJ|y6N|aU@{UWz8n3Z8?jhufI~AevNj1C!-FtU ziY!fbAb}L-TDKzaBnqlb-2}o2N-jKigcCx_J&gLk$cgxj#`r9Uq)lZ*IgzM6re2vn zYH^0DO-V&G6<#}k|DQDQ!6OSzP4Aqel#ZZVr0VkxOB=uaW=!D%uLbO8)J(>|yj9uK)Nu_^k-RJ_6=JYao5z&pvRI^@>I9D~f6}g?gR}j&s9&7BzQ0hED-MD=bpiS;0k)#^5 zg3`O&5iJ|$dRFe41Cw;0N~E5PXudb1@>%4nt4&nZP!p*;DqP@nZo+U@V8k(b!P_g! zK;?+jy^l6Y5*nr}2DCx6)95j5(A=CPxq79ta!IBrnL|ZlN$VeF<#_83Nn_i++{l#6W()l z!Y~zN9o(pCok|u&M?;2>3+i|h=2StE#)z+v6rDyCFftlp;IbPUbcS4!TJU1w6R%c!X-jC834#`A$4u@@G9PJET1(1{%(s zkd%~Y?rY$p^`hy73t#oKTdpoSfus~iqM~e&)iptGlDKl$coYy6qpS>yC9KQ}%>BKl2uaV&pykR8matgkYGgsiCFDi7K0)@?Z%@aB-QO zV0`eXIw&{B4s4)ImIPne1g^##cgbid7Y0p=ladKF>=m=aC= zBqK>uO&=)cvhYvh(@TD`VoY?sX408>3&|BVDJs^vCTe@sdr`w&N4i>E*Sdc1dYyga zIt0Hg`IP+=hs7E1%)vcSq-aa?d(*vQPBI!DKfXQhvoT& zZhG&qzvKSqL4YIM6aUj|B4RnO+kf=>oBWSnnzhhcW}RsbBF<~A7g>LcXy0vp()t== z{=L<0%dmNExi+7z8u4$jon$)~y>OH5e%mhe#izC(Y%%r$b`8H_T84gEXm7Khg5J5p zeyjZt=%aV+f5$7$367!Ish!B*FKcm}#NRKw5x-PQ@0(GX-ZT5m@gw?iknMNHbh+!bu0S)wy1}rUO?}E8D(`PxrVvM zxcsi^7zvAAEAXGZT3jc&&UIaiQL)qYgzHs|jmKTDxIV!M`IP;@Vl3at|L%1N-cO|W z&6=WD;rB={#>k;J<$mA`@zdG0SV8_ix{E!6|M=A|M{EDG)B7jZi`2WQ`&Rg&fJFZm zeX;9XzN&TL8ceXNV}RT1POy#&yIjY)%fcu|b<-th9zN+?De()~1?&;_7=)hXf@iu< zLVhMne?q=VN?qlq5@Tnu+ag#mus`g6Is%=mz`i(MhQ!12UH1<{q>0}pgnKz3$2~F& z9A_@@T8QvyAg2oVl=$9P*75SS{^95`IFJFgm zlB}2hKgTUh;q>D<{*z;7%tBu7kk;E`P6nnqR?Bka>=eDO$b6!<^~?h20tu0YLgU+A z=KOxt-(Aa~^nP`98G;hq+O59pIpv;i^rvE8jrpV-m9c-uIAh~tyBQ864315(9w?>q zBTA3NIQDbSSyFd;Y`VP;l;dJg=JfMoFO!rnPxJidSTUmRiKXj|5{-vPV|Q_! z;a|E6{>!o7$Z>oedp}m#kRm)gj@^hcC1giBxQHqu|Tf zY4R#G+cWTAV!JT%L^&a@%V4lI_@w}&c(Q${eE-~_8Q5`b5x-{{lyL08xD&%k>~7S} z*m!PtQrs#wi(_Z8MLpKEJObx*6Kw;#DDEm#nX0@k&S=L&VLalS(=B!p*8!a^7k!gn8lO*Pl58^0)lpW7bz#O%4Hw@+4zli(E6m1o!P!m1JSzSSAgepWV z^(AATU%DG`$BS8^JIA01Kh;VgBi`VcF}iDrxQVlwN4S~S9cOG6JBwYx9%S!e4eGV{ zEDP|K#2U-_c$55D%NM{BtZQ%<$1vc2Q(4h>FF)FPJbs6N_jJtCJE1Fn!(UK~ZMC)~ zwsp2^ZI9bN!uttXc+22uyIDiwD} z($RqbBzO_te28rG@m}xY)VupTMQ|W|BV>2W;Vw)4aB~03yHz>=Y8n4NS^tcOOPMO8 zFQwrtjZeqR$rC`+oWosq4psJ3uIE+AcLVvaeB{TKvxsqkzHA@_%a>7 zNhhL#&%jNKXV|4?g(iGT4m!6;_3G|%(^QwyB7x!K4ic0JiVD>!gdl9hP~6Zudn%c# z3KNmGd(L!Kp@_%r2!oq=B*;%(gD_DFs}M!eFtotbLTVO8grvoHU?C=BtoMpUunk1< zres%9=atgQXbGgLDitZCGQ&$S9RU>42p1BuT&u+ z6^0ZHRf#E;Zj62rLbO8){Zbbrt$(83r1Wm5PNPst2_2zwNFjoWo@IL1>}JYL;HoB% zBKL6b6{yyj?d}Gy)Ol=nLr;i8o7_i|s-mD5`)|;&)U$HW9MnU1C*{R%RI423xf%r~ zqH3Rs)EyPo8zu}#1x6f`PrR+7EShcRi6LrMP^y?A=u;?W%nVY`G*>vpG$PgYTBQlY zDsdE1HIWFA;0cx$s*6g`xE?V{grrnUI3|QROL?a(QJI952Io6?n%Am)0PPi6jSYi$ z*&6zZ7Cq5$BA6wKs(d8MQ|nf>c6PKktq#^TH!W^i+SK5k;B5}J%oyn5pLtDz-cW!l zmjb8X*(r5aIK6BXK9?(XP6AepdP*y9d&IT-jO4F;!NZ-9d&=G=^%F{wbB~$qb-8>fcbO_Y z5dAUxt0d;j^{_?24?y4KBG+U@q#`(2z zCEB=vHm+0~SEh|C*NSqrqA^-go>r8v6%}Yjg<8>At!SK9RHPLZYejyos6;CYXho%3 zQJGd$t`+BM#bdPMJgqoiD=yHA3$^00TJbooxJWB5){6aFafwzO(27g7;xetcT=VB@ z{xOTKO2QJWngn*UAgD@#0L!2TFzLf3Tt@7%VICGlUWLNJrt)Io z^Z}aU(ADXIHagWeVv1SS+1}9+T)HsW z)(YSVWN!I3iO-X_BPXYJBB$lfJu(*_VID!5@cU){0CIBgA>{O7+T+M+j(r+A*-^Rj zF$>BR*JTYI%i8hru%d4RjeKpWTh$I|3DUWZPMvKyvAN%gdgdc1`}N4Fz9!^UkK(t4 zm6iS9K}G|pGK~#$e~mzy`m6hVytJvsd)VRLrFAQnF2$JYROOy*{Z>gsM&jd95Z)sjs?f;`O&a& zZw`!aZ@PyyA*V}_=9mSQbxq5IrF9*_+NPyJ@4;TmN`uWEbql<$%Y$u;np;-{F`3+X zu!qsSSdAMv{eQi04;1&Sby19sM7grn+um8<=xuJTuWR;pz=%CQA=(-n7a!%0iFG7I zyKQmt36_}HIA^pwW?%7OZok?mpW*I)^Iye;YEk&UjE~vo%-R6@i4x6;1WPdnYF4#l zf~QU|Yinz5W2fYJ_}XD1_)`ItXg+cQS|H!f<#`8t7X}x#wgtU2TbfpSgUed$8zH0g z>wwd}ys2ebXNR|`-HWGZZ|fp&TV2cIU`wlaS!;WH)57MUw*f=XTi4PcOfIeKz`~ah+kRLbkrOrK7H?r5*aq@u1$|t#7PrtE=w_wzV(A z`lYU;6AlMa8(iTU9c_4&)+=0C2M*b9?_9R5wXGxAK=nb*+uqRzDLiNg+gdQdgA}as zu!J&%I$PR;b*PFOM9<`S7wP~JN18ur4PeQ6*o$c3>X8^0BogCqh3wU&dHNzb7oZU&73o^A4%O7J$(q8K0a@|OYVzl_nI2W{NZ@9E-)7@}xjgucN=FYD9q*3!QWqc5YxBPAfw zuQ=1!1n3(K^zXw`kOm;(f0WQaDM_WDuT}rXi+om(qN<^NJEjlNW+kZBaJ}H zLeh{%B8@^Cjg*aa0Mda-2O%Adl!J5#QZCXMneveGkqVFsk;Wp8Ln=ZlMjDTFDAHj_ zha*iunutUfz8{G$;Q&%8QW;V?(j=tGNEI?oK{^6yD$+EhN~9{JYNY8%GmvVKXwPgW z(k!GSk!B;!K{^WQXr#GF^N@}~nvb*qsSXKUaxUg|x{sfLyArJr_SH;no{7S3{{LCL z|FgeZzg44e_{4#H@6z7QJdEw}+#}uItLL66jP1pf4`1Ny)zc5UVK1J4W>5(B>KSMj zKCRfR840_~dod@G|9kZ;bPQvA@ier6>ffs;qK30l#U3E-CwT##`FWjV3xX?`H8<5a zb$FN6wKUZqbC8h9!Uacbxz?5-tsf5}d8JR!M6^${a1||Q+uE=f)dGDiLOX+1%lL;j z9B&D(=vcKZNbAYY`i}M>*0C-1SePygwsbC)Ds4@5Eged5c^xlybS}f%QWk`=Dxp@f zKBqPP3amF*0#*TXa_hVcTU(ohbuDDMPA+2EMNdEZRg`;ZgL}vI|2#YR{=B90o_hJ9 z>QC!e`hR!iahdCO*2zV02d#@~fs6(GQdGYb%iTKOl(rxi(k)o{*0qSGuy;jcu*KWa z7{pQ-pFcIPB2~BQczhOyxupU=o{7DdMC7NyN7{2CzfT2z8elEJOsDP5 zGXQ6X(a%EpY`{8z_%zJh(du2%)`ak{XhfU63p*Dr3br-GCbL1XPkloPRkJS8lrMAe z{F((?mHa{m`9iQBcEw5_%M_Fe&cSI(Q%l3S$d%vc0bd$yZ?9V%JOV875CvI!ULrf^ zlOmF`y8-0@c2!%K@<#|nw5ekig$FT5=#~ZBmNwzZ2%qsa1Y4Sd4S1kw3D(P?FY0U$ zqqGM*8e7{t%nu+f!HyNJZA;7~laacG{IP~Al+<8DT|+|~nt%r)FDg(a5~*8`rz3i* zQAQhDSF|i_Yh8lQUe>fMDBZ7XMn^ZS@&;EnwRg0)q7h=4tZ40QZtyO|p1=}r4MU*~ zjcILLg$Keqqok90Ac9Mmb*v(T2e-B~uktR!KD2!5#G@G^*M?Yomj)Y}I+vPw&8@A= z=-CaOz$>8;Ha9KB*c~0rS)Ai-U$qp^c}uWC(Yh=fpNwLJs78?$ZzG;Tn}h1HQ9Mm9 z!?WYkARh4)84s&y8S19TSuxmFV{aVKv&ygbWp(wz_F!;{7xm*g5*U9fq&kl4Y{f+w zTv;CsHUt}ZtsSkc-sZZt#dwNrZ|!WWr(xT^D!4LO-`Sz-aasccLV>Jr!;|yD4Z-D2 z5aumHquYWl=pejEQF9$N~S#ia}_DX4xGeHtwoZA+Jd_MIo0-}mTW zj$Z&Dls^{&j~(e@b3xw#pnVfnJ`Z>)U=c=Mv$w4i_X}*(@#@EW+g8wspgp%GXbbI( z(3nJk5AyQs9GJVGigdk&Dxy*(x?d7p1Rst2EVgjy8V}NTPD3kh@$e2Zc_L2O9pcX#~?k3LgZc1)X|8pomo>mzr-InqIQNqPU{iFavdM{p(l`5Jqt)JOE0 zRQTE(;bEI_+6>qNxEMg6#ase72_W3=(YY}TaCt3n!mWf~qI{mG%_Pie4NdLKn(N?= z;jKHgX%YHG?vQmfQonjDO2>N_;$oqD6Yf8ab(ny#soR7-T^g^ne@kO_DY^)mJ}$Sy zZ~A=cQov<^%K>WMy8`%?fbD>*0KWzN4sbQ#u+A3h^@a(sbyAk7V<6}`D4XZbYe2ge za2x*o1a`3As^fSUj}185JD=FnS!-wN0PAfHNFS02`|s>SPXSv5hnwjRyHNNZ_r zIT%+p%?DVE@P%BCep%@Ky|lHVY0)aaa^WK0#@5y)-bHN~)OA7$Gd@;9JdmhgwdZ`` zY7D5k{Wkcc_RMKAE$tNZ;P~-a%t5{h*Z5@QPXK5g*$`~6Z)@ULCSV0J87q*P#tvwu zSb&%)hP+=aKTwmT1snWWeN?o}Y!4CwhF`5Yq@7YMIZF7FqZ%s?zgTLN@s$Qim#QU2 zMZ1CPL`am;a$*K9gDR{h%5h5)iwOgpiWNZ(EeK?V#F@-j0~KNuxq|kvC5}%@>S(vT zp{ldvNLpoMo>xxNHbeVIPhlLS#p z;VSiIfF}B2dnupk+o3xRQ~OjbVcMlDq(Il1Ury{(F|ShxC-91vYB_&b(fmErJae1J+PxSf<}$q}HnU?<5vI!4Hoo0HzjN9Aj#e=_ z!a06L(bR+u^zrUWJU7vO+h31~znv=yd+NHsANKU?hd)qYElO8kgS?RFc8JaW4m=+1 zz_m09>c7WCL36(OzVZO<47a$Vv*WOLHz+&XX|)Z>&d!L<&a`J|j~bDZot==B?aj^} z=FWCNf&oTmXQyRF#RJO%WLskBoS2@SJaF((eqW;d4?VNcb0Ps1ZvZ|7{2lNE zz?GWJQUG2+9$-9RGN2Yv4_FRZ16U8Z7;qKfCctd~6-#O(;RMt+DpNbC&9p{RaUz_6 zVn95Sr8%nzpq{%)mSRcqAba$@N3sNDlhzd!PukNY8p%*!kX?!+;UuTxOLYv5OJ;`v ze1I8%g@87|TEJ$&Re&9U#{e$?-U56DaK$IHRDcF30vrR_1h^USDByX(>wpgd-vDA0 zl34~|G@uYr0hkVG2CN304Y(X|2jD@#%YgR*p9B5{sF@iEl-1Pal~hdf6Xq{1ol#La zv!;xY%BoubEPusRe|cq9&CKfRsu{KZN&bpT|I``a&nfYjmIJ2MOe&dKGbiA$43teR zD-BGpm|7aBs;u;vKpthFGs?dPHeiO)XV3qilL<)wI&-GplOLW>?hI zmiuQ;tu2Q~ldEcKXF}Nzo24_RRZOiZn_f{_J)>&U4ES78SqoQZ`KMM?Rn$-|6*Z+5 zGp1rods=zL)Us()D=LqGZ#5N1l}&@eipf>gmHugEwN+J9t12f|!i%aSE0MugSuJu| zQza=^R+Y>wuc@pmEt^$YRaRMx)9kW9W!2Qc5jeqcW!1E@X*ktPs;q)TFj@_vnrg_@ z%zA!> z%VtcguBa~Kb?}qk6}(hC2NM3uNo6yt%4=#!vSvnLGU7V5thQoWS!vbG+S=K*B{g$q z^QP1W{MEenR=M}mT8}j=K8DQkicfRB^l*bUH|^QbgBw1Wr#mh!fLGVC1v8e6Yqd71rXk$+!s5(X{(RH3i$2Lyx9?xzobiFAWPif2^l9Mh-2?XiaJQ zXga3?drBQmi<|=HMc{u0flSw9k}NE=OV51zroY?RqEE z#cy;pd1Tt+qw5kI(B{6S)tpHpM_;t_TKFzt4%X$^WgyFVFru{~ZHn>M!Wgfa7xa5h z*zJWBbsn`x>QuDUEyK0H?EevW9`IDY|Np=5GdM^_Nf{9;Wn_;el2l4dN<(B+Xc{36 zMKnYSQ3_>`qJ+$%q=*J-(4-=(QpxGU2z0UjG`&#$=x~}`W zZZxf9_f(3-iMzQVJbo8VR0xiUEB5&hWntx2RHOc%6h>W#E!CorhhglgWI(lsaDo}( zx5FJN!~+c`h+pJ@Kjjg1b|{>NS}v4_>NxCX)j!cQ6ybk~t>4%4XDtu38=#I#^(#<~ zK*JY_5(76KJky*L^9Um-6>GJ|bmvmZ8ayy$-bx>UWM$N3BmSqii~$mT{o!($w#i zQom7tBItH~J=R3zA=d4ey$NA1t&i*Oo?1x%L?ThkFseN(Q zrH)7aPMtgTopXF@J?b}Vd(LkhrPimmr)=tc?8V0r&R_>{1N*>H5DFqe42T0M;4;Vt z`JfCu0x!TD@ENp&Z=f6W15yJ1D-Z-CKnlnLc`zNQ08O9^W`nuF6s!PNz#eP`yMZ@2 z2>iiua0-NjC=d%0KnA!D^1v-{7gU1=&UBB>Ix&F>U-4l;hgn3>6BWZ+84Dg zwLY~^3aSo5Z9~NswTxOv21pNor_O=e2dBP7?VDPUQZYcSPbqagYQNOF)c&aRq^^NF zM@p%FFf~AJOC6W0?`QxyKy5!AP|K+GIp<1kN9~8&A7>k??nE7fQzxU&Ycg1ea61K( zKpwaUUV}cMFcI+%HUKAZ2!w$okPoWBYtRqGq)-O}E5HWe1iZir5CgJ7C3p+If<7Q9 zjkpDKz(%kWQ2RIp4Fz$a1Uv;@VDzLhge1@gD*?6cW~e7P11^Iy&;Wh`1sS9fU;%7F zILHGvU;qfqBE17GFb}K;-XIWM1XlocjACdFcn`jV1sh$rNDp=O*rBLIpR?ER=WfHM z39{pa3R`)wY#J52cxH3RpC|s+cO#64d`I@3gBZRC4BbG{xaf+Km%4>gZGE`;y197~ z|M2{Y|8R%;{fzHF+?C%;hHo2$$sZq;`1{?GxyHML8}*JxtVP z{PF0Ck$&iK&mdLf`LhfZr9p(r}I+SoYLB*+=5OX+Hl)rQz;`!kb-a5N;l^yx)ULUp(4$p-6 z?Jlz&Wh(iwJDZ2%jyiL8iu&W^kML}Xf2sCSTl}T|#$IW-9yFBpe!KsFj!pHIx>B3| z&3FEjZ2qkdNBpY?N-T5vhhsM)?l~1r;xB&l?`J5`^@7tF;hv9l&53_j(g>>67ai18 zZz(DouBa-()5JLp`|v|CMg_FRd0M62*v`G?l2}J+o8&&~pa_yR+c;vjKz|Ja@+} zc%oq=HU5t3lPCV8C(XVm{Q2WfTS7b~-!X15#*;zJHrhDGSKt;wWS+I#zZf9nee6C5KSaEBb2d$p+H!Yx?~7Z!aPUL zrJ0aFnDF;8zJT+ATThfw{{H`Rt&daxPoMv?=s#`pKl(k1WZ?O52A&WnZep$v zl7~i;z1-C#lT=lr9vsI%Qjz52;^iYrS&~;okaS>?yd+~17ETmj$2XG?+mhqQE6^25 zZW@zk+DvU~e=G(`=jNsHlLF*SlEGk+V`*YIhAxR?V5Ss0Igv&u`6!!7;;3V31li)7Nq(j;iQm3np4>K|bOgY_(_d z(7kXT3>@#Y5Ftn>M+wTnPR1p3VGCJY_Y6{rHkTB^S-c~|m;@I&o#Y|tICX|Rok7N9 zTEZ!$7{Y>}jVCp@O!!n8-lUemG=wxST@~j+?VXoYqsuW#)@*VVO@oJ*K_ji{I1R)F zjXY1%St5v8Iysq}&NSiTrve-2XThR^QlWp~yM7 z20oewLY+D;A;cv>jH1&CrXDvBgP_aP2+}|ZsfEUmYsOa8@fd_MO_ZSX2-3JGGD7Ku zHbad?jwZ)3NdbbJ6k;ET$(%ijj2TUk`k2^$fcA|#Bw~z*B)GS+P&$zlZG1ZM_y8Ix zerLc1d@5ANkNk=YVJgt|c&3jhrAKBxDyvX|fr}=2|F1G1&VDsA)LzW>hSO$+&4P}hKWQjRUf@bzFnSq`$rK1{BxEWgNZMz_`9HD-Yh=dwPU7`A z|BH+wf$YFojv^f)KxG0Zor}f#Z`r_}jNxWy0>+{rR*s7kqBksiGG_<PRa&a^t6PoZ9Y+`vhSC#g69@}(K8i=`w!}b8Q}>rSDB_S4NeLrm3z;jn>lA)YjC| z)YVeaQq@w^QrFVZqQ-A&Yw2j|YO83gYO86hYinq0YEvV+bhLGKRCH8z)O6H!G;}m| zv~;L3Ub?zCMO_?T7rWNQX1e%E3FD|_hORjj?iSVFrUE#>RSy5gu^;@+Zupy>_cuHL zZ+5}o?83jH)zo(boi4CzB{eH?qv z-)y75*>i_&MQL6|UU_-mX^J=?J{-UOF}%hnpaw{LOV5;+_U7Fuy>FcrHJ0vw^T9Fd zBm~*d=$7I;^;`(YzWq159X9oRDrY%mQ|+4^`!l}J1pl1hW9a$j+0rWiT(o@$x{Y0S zcwjpv#Q74aAFdJq|7Ab+j+?1xmuE?9Q0Za5v>>q#x(ZAkVIRSFs>P49JOkgQN7#4q zooXB8Ebqd1s!f(-58^x3mWvOIWYJk%f;@u!g2LlP$4icvlU9&c#|fCoFOy#NtT(>AB$ zPAM+;T$){Yy%oI8ym$JZ_RaCF_oW5O2QCft2#pRc5B(Y?!5sR>gd0I)a?wzv;T!dZ z@?@EG9s*xU9`?^X++5TG2Fhk?860R7nABqU|J-apjtkE4(AMyENbaG1z~N?6@A9Tr z<>sRC(0Lf_-O*VjgNp}#3AIxcZ75V(T$Cep5=3=23gQLqn2#t;k z0*lEFuZMl2B<&&84;K$sWS<5d&J~S^z~;EC@(f=riwi|R6Fb1nfjrEi>*hi29quqU zR%0@_SwlgKt1zb$TH+;&WfWgV2xP~XoqH*ycAaLVi z2NHGB+$g2tsxc5G1UG6ORCu9ILC~=y29t}MPRChLwF4^XP${AuLQIdy!pqTdDlCF) zC>XhE+z421F5DMbxPLL>dgI1K#W&ug&dr1Iyof~}>aN3sS5={!z>XKBOCA>Gv@#iJ zYT#mUGjY_RaONR+7(5Kzh`G6G2!9%O0`GY!$uJNO40s&`i^YYwfO~>Na}pAb#!cr& z3}Xs!I>G~CjzGi74Mha*SzIhC#<^*TAua?zLJ#{MPIgQ#76Z3V>=)f`ECOyPLJ@`G?441^8U4uNEZ3MGSwdpL(H1dB@AEF1)hjtUHHO~dGG8e(iH z4{DKM$@G36m-s zs1%IUfUL_x2I5A+iQF-qcuf<^Pn|yO1D8miN?LPt@~@$bLNhf@4I*BqzyDRZv#d)X^iB zV!kK~EJ_E|Hfx{ig*-LyYB$h}<(igaAk^*PAigQfGK~9&Mj-M7CJ15TV^~+>eoL2dgY`=G} zwjO%7;7m@$2{Yq8@7_;7H;yGMF5d5Qz%#=sa9hw%>CsK!J5>gjMu)FI+`+o4nY?_1 zx>8}G?EHdjn-q?}IyB~krjMs$`tCFf73tn_$5kt@Oj=feKa$#iz>^ znZEXHIK0UAllgQ0D;G7=w@KDHr>O7H->^CB4Zo(B8$LD~zZJJaI3%WW3b!EB&Ll zEXcdr-_fa8^YzuH$>N2=v~6w?17W-4{Z=os`KgkKVl@wS4$VK4OY~>SScFAnNFP09ml?RhRr*4PriGRD_ACqAf`%Dd zCzI|-`le|7JUZ#Kw?b|H+$`arjk2|*duwyNlAeP_N`~{h4qU zwe>qgo5d`1K6B#Ju8^3|%3IXSj7|N2UfOS|8nDjzwB)2Wy-w}d_AI_|xqpj`NUr^~ zXL-yPn>DZJiRsg;#Xc0O2Tco*yRjqHS%u5?*2CacZ@Rr|vY*6<`xgY{eYt&Y>v)fd zqj3v@XCI=3OtBZ9M~CQCxK=ziJRyP^X-Q@OfQ7Vz2? zZrkU1@Wy$4v!|@g+834iiabj`cb-0E(s?*O=nl)~d;a7}Q`&T7Ew?5v`TB!5_v`d4 zDLdr36}QiS^!9QV|E4S7#qQ7Z@_QLqdC08oWO?+7=gVfDIpZK)b>LEr{PjHUF9J=Y z_Vo{h-?FYOcd<=g*k(R)X?;OaU#7?49)bKZQJ+3d&g9Yhsb8HEoz}cEaO0^=56M8f zJHKz(T~RrwjvL?4*G}GH{P^MfJ-;N=jh*FPnTtH`IQiOEwVqRJ<(>O$^ZO;WyXVRE zlAlTp5+9uYIVVcySk0<7FWSX#0z$Vu-B-+Yermi*BXG{Vh~yMc=c8kIeMdcYp3`w^ zRgrn^sxs$@O4;CwG9hQJ*0!W$tvUWh#frKH5}#=ouZ1Q*{iZgy?x#2R()aPc$KKp9 zugoxbUitmSJ(=~va_N?zwhTrE3m9H|R1_BZTx+ZLsaUC!8$2vQS*6~=VB`6RuOCZP zlOM%flltq|;5CuR5AU`vH&6>6b#I(r-+`0qQV&nO57~U+d7}Ph)86Q=h^fYzQ66(% zR|`FelLTM)g0aV1+Pjps=I{BFwJMkwABu?V z@;z&zD!+dD6Z-6c`^M|m@}Kglof!FC*ssJtbb!~(@>BMkW19wpMdMwPbc@W+Y`1+M zp=3DdYE>01T&^@ppSL&YfW(mlTY7vV`OX(;Z2GX*C0f=gnx-4;v*h;YTZ2*(O9Il2 z=g$qeCvAIu)78PLmt>ObL?(=F?oGD&8P40eG|IWk9I3^n^9kdvD*t}-3fc_Fk$9R zmU-`hh0v*c0C}SZv6>i4$6nKIq}ovJxKK*0V6jdy~}W zR-a#0eqH0%BzZ|Ri5Y(Mnff;8&|}NX+B}u2!GcFlX5_YBeZ<`nd6>^&;HpSUR{e)W zDgS#;#OA81!q;bO-kyGQDsgk6kqt2?_UZ9Aa@*D`=9j-R#TR5Hn>)Mj z;cgS?-S*|KS=Hyaq4H(IYZiWbQrP$-s?$dO#QNv?sdY-b>|4)k+*Z`}$+%XX5^m;o zqHyz_Ip00z$JEF8TJ*)#b++1wFZ`9F*m%^~<@)*C{^PoIZO`S@YFnk%R>|CPmzPkz zJM+-wio`ry`ixaj$cRhyRY z{It&N!EvdOty7gxZvOFF&1BbeXP<2ImzSPN~-|G08=+$W#TeOLN# z`X-e>xXR@t{6J>oRx5G+%(%xr2d!Dst9_ik?8H>d+7n8nGW3{kS}N}=`fLh4QciZ2 zMjhi4GVXg>q<14tq&DkIm$A|U5sTiC_B-E>D=~#cBTXgDltdcD-LpH}JxyxY7hBoI zc&@FnecYHfT{MDMqeEkF`M6z2d;LeJ?HL{I?-LedrhoZ^rd5sb>IsiclHTO;#8$4c z@pU^|mlo}B`8c9^(LIBmI&qH_xGH5=diV~Ws&1HSa*4~%n^mkh&BFdbwoz@S=;Crc zvHR^Fu1@+=ji*g7NLuIJm!AE7`Vm|I(?L<~hg`g~_OCo~RdagpW}T~ZQ)HK(-FTr{ zRjRb>*TJ$^HKDEVh#sc>{%^l@W(mzY&S-dY!mz;i)rOrq^N)-0U7lC7Aahx&T=?!l zt_kfgPCQ@DbFM+|X}aD$(YWfGr0#&HO+LJJ@r#=ZBu~(JpZrK(EH&xD;_B!P3JyE{ zDg)eVq&7Pe(xvlv>lu_ieqB@U=lW(2(e=D_8c(&yT~XfN_>-0^h+TqPx++U7Zk(_5 zs?KWPWO!oipY!=3(V z28Z8%=`fVPb>yu|*vg-KyMjKfX;*l3!}ELK?1s>@RhAP}y4~^*eSE5>5;S(^E%J)g zEUgtz?_&cE3Nj;)e7SXYYDMV>zy z?vZ7WbH{&uqi9|_`d$TU2+B>Kj%t+qObTD*U*jIeboEL?S~wM^;B@fnrSJrM~5Q z2OiyF#4GxXto?a=!3X^@hdU1S%6xq3Y8G+!`wMG+%T1}5@;<+L{Lp6ht!s~R1ltY5 zE0d-~ax;IOP?aic3T>{=A6&8Xz(Ny`Fa1x>3G{#2SR1XL(9w3hC3~R!kon&2dH3($ z40di-j<+B1w%a1k>sa_J!?o8sP-@4od^c9dRvd30^g?wQW(Uv7(#c;mj-WLkK)!&n}F^%X~pWG%;h zIdd)l$ALA+H{VQ9XjfFWY}*&(WoW&`>Z~PkB}tzf8kd=f3FnPz#QYRy z-$FKtOl+!)3I8N2S^g_eI>W=}fm(;Z+=f%i!jkoFW2<;`eFjsK(yw37TO~IpB=R8t z2bp>O%gpaRj@Ww3-`K-u zPp$p=XE{mU+{JfyeL8NscPrD{Ho#?Jx_zC*d?6*Pb9c*r%uKF6_e`I^F{0K>_YMjp;svcCkOFRAk)5T7_FUZ8xp?i8{HdBz#A=U~ML_lL%8%(23WmGP@p9TUoSxAY!gn{6@c<&w{<>CQ18W1D&|f01?O3z7NIcKh&= zur(KMyl88k7o?SH-F(j6z5gdI>&V!w&*NzoFNF>!J?^_sf15HPc1mJo=`B8zJh?*e1RVRCt4*a;-qb4xJhdAnT-&w=!*Tk%f z8-tH$#uzFrE$}~Bmop`HAgMTDVY%RscUoN;AJlT?qVLspDeb%eoPVyWRgA%hwz7PF z7cU9@Wv4>-xgDvtPVSz5s_Km0;yCxcRST}aed%~EPwegNqAULQ#A)YbUaT-K*|IUR zDt+Cj-isx3QcrHK{aSj%z$nt?T~GW39_KupJ7>gdHs6a5cu+53Q^9P`%`TL_cPl@j znOHeSXZC#A&2nv9^sc5f%aYUh-usW*eN!uRlEW2Vm+Y8dRp*j|PwzCo;W&HUiP4R$ z?vj!a*WAFckBre*RfHw^vTxkbTky@J@MOt?9UBzBmSxH(nTmDRwq|`3rky(pV(ZGJ3NmV3>K(djyC_b-TAt^Fy@t98MZjOEYDI?u25RWg70>#~(tYd?g;`S96g2eBL{Bnl3b#J%=lL?YSp8w43u8sZ ztGz#O#;;D zvi?NAKC)@XomVDXoy*=8B&IN!jIwZ?(0{MZIL*#Qu&3)oTtn7KMrwqjFgj0X}Z8c2(51MQSBSM;it9H#S=8MWG~;zv9tBgYJUIu_`b^A zm@$@(+j;o!6TV-s2lV%u9~r;>{Qct#_P;rm^T_SOehrIi`->YqUYxhwJ()|m?83q6 z^Z`??u;q8!pIizxV|4xTEv7x>{(3(hrbsjNaS}N ziQgEOz0Y)F>j$;158pSdDsWp#h6J^ToPHQR*H&3=aOT3SI4!jlkKEFLnw}d2MLg@? zn%yo64f~LleAH`8k@Di_X&Ezr?v*?##QS3rseN8R;MsZ}&3d`4u)uE@Zu$9{j~Sh{ z$Y_l6+q@kaTwmiI^`Zcf8BBG6=tEfzhLIAl7(jt5@h}6i_cHHlekjSMZMmqT0p^J!aGaRoJYxV zOAN+;X%7cK0MK@f&H zbMr>pg7GzX)7(8p(wXwH~?-O)E}o1MSDQz0q+&iNr*x|8{Sv6k=z?K?xZ{MIt@ zH>Imy`~4WQ)$Xnr+B!3wZ_HV?ZFpeTH4*N6-c@IAvTecWMba-9|`&(ZMveEHwwF;RSuuFB9(0Ez8IWZ-V`h`5nm-Z%vsSXQ6vaKVZ zrudxi=#;tpegBP|$1}~0j@Q2NQ%L{Zef3qML%`8Dk39~LQb#Yvxf!32M0Z(K+SMjh z4=$AQEa{!>e0P^~@vLJfOAkm)9rtZd$2{F7CLPCD&3sY4>si~GwCfe`JCD1cX)S*^ zH)u(7q3ER|iMI9?`cr#v*Qj2!Yin54ZkLpjxVT%QSI&CghES&tx7o?}KLu+w>b7S; zb-!V(ARJ`g|L&TxRe7@g;wK-+Wry!dDtKvQI#aTxNTBiL1JwbOW{aZ*4M*bQE{~DC zKBuTK!Y(z{d2H$fv$K}=?;a_&?VlAMUe-4|0t4%*&KE|;?NLJ=%U0TwpsxAC2 zYOLD|PgAF7@IQNek}N$LXTbaYr9-M%5bLgQ&l|57Wt*pXtuR;>n|VJsPG`;dQ@P~M z)l;fdc{cQac;LTd!}Te&#ciG&`O4OA$neQxu2Z!%@ZU<{Bulv zU;2$3H0juT;8V5T+072)?S(&Vw+`rLK@3p4VzQ1peKvkmqAA2-WO zE{`QOpPH2Ak|Hbn-yXc`5YJ;&a{SrPDMn(+d)LZHS#z5>u37!@tfgZ{^9fIPb?u{P zN@pm=-n#57VW0VQ1>xtsFU#AL=j+)x*+Q$p%{SHTs+O63o_FSriB@=Cj$!v;FBvbr zNwhHHS*c!f)3WTrJw{bY23D;_RawWzNHtx09oX!1Z+_lsZ8B49tNM;I(N5X7`OlMb z{pMI1&nNhDC0n;XpDy_1L(?fYku^8@6GFx`-W6+^dveChE$P}36+`lb9dhUg)WyFVMmk9CA>@*aydD^dO7;Vyq?sMEq0>k zUj{JfPeVeM)pbA3HkxJS_c(c8$(?iSLj2eHUrnBu7b1I%Pc7%rSFJ5y6DN+X+v1V7 zHE7UY_KGpBXOZqcjaXBYXkJ(z7BbajTH z@bWw6n&o%XnnFVlh3p9I%Ac8&WYqIZR4Pwb)}x}%^SYl)vSU>5cJr}}-O~s8>)uVS z9^+Sih2d%m_ZoQ2=%D)#np*a@ z7xg@=?h&x)P4U}YY^9u+(ymkO`^$WKUf9)`Gd=6Grib}%{OawtHnidO5q({^jqZ7k zzuM0V?fMle`Z;j^qZ`7_R=FwUst4M8+J$o~WwKIF&Ai5Y%i@h~T8ik00}8R@vUzOG zRGlNdSBIW|ZM}P8)O)_%nEu22*S75|`P|!_J|~FV)vstwu;o~ZYajO5Pk5YJ9z6N- zv#ZiY)XI;%c}FSaqmfag?X!s=Y%Djo4TcOkDICa z)qR}OjF6rupVpr`bM)L1qZ}UPy;kS;`U$-MBOSx`RnU zyVNWGXe<4IHD06oWCmA%Zb`Iw#C3gC^0SJ72KT{nr@tPlHIn<1$Wv9+ywbe5K4-?~ zW9Hl1ghm}ymYXi$@uaCIep^!cmT|#?=U0h&xyJ<;1Q9m2ozBloSh;DA<&~>bi8GJN z_vkn|K5Q;tuKedii;cO=)lIWk{Gzx5rVOLZ5%Z9NJ-l#<-{w_H0YA1r63#Cw(@A`}(Sc0sme0C- zxXy8VH`k*Tc|~NMmipVMYg1_R*2Nui)ezliLc7`OXOm$x&c*D~Nz&?99j$X}yyyBS z=c=SDyXIv5JV9?&tqNB!H=<1&+_fUseHUZ%{yDRy3ms#P6I16yFdS2^=+=1@a^0mzRXwVlf1vJ9T#D`^Y|Y1JuanBlbY9Ld=W2N zzgT(A&GAd!MvbxGDbaqh&Gog{j?P!MLLZOME_+mSI5bHiZQiPihGTD)tF*aH6K{lQ ziPkC14%-%)ck51ogwQ8#Mnh%xiJtP9v5_xpGI>v3l;JfDiHMRcUvog%bc5OTMX946 zW*5;X&s32*Q-3|?o#ayiKl{tZ%5wW%UnTE~JMp;b=;MLbmi6gYf~&>`@n_DrRt%V( zG}&#ud#H2dmt_qTJbubA5uWjA#=TkZ2g6q!+mUki`A+A;L*Bi7^~JM3rtw*jf@Wn~ z?bh~;e$v4=>DiJLa^;(!^JjUkZ+`EdI5uDx??H{$o-~K>Ywy>aKYE_TZx%ZzQFr`- zy)hCRN%I7sEQ{e07FzIXqFLi9>7xoQzG@7URNgkz(>7MwTb#2S;|vaqt};uK?7q?U z-iWJFvMOlP`BWR>$y(n$tD1c;KlYhmW8}P7q3)K7s8OTbTJBh>Zejm*KNer@i&`Oh zS!Ln9so7^Xe^r0;YL;F4|#KlscbAE|`yG@w9RM3+2i>zZ_ zr*6NSI{il8*Mu(@Eh~<-)?NSpGSh9d;}yZn4x+_Rza^ZCoYs-_CU)EXG=qlil^KFR zioZ>NV#&+XFB%kjY_LA`TSvxs?vBNa`Wl``WlQnucZUW2vKJA&QfMk*9GAFw@$URD z+C^eDV%yG-u{iO_`n7pW+nsHbj&3aXQ?%%Qch1=@BF?ALS#SHHkzu(w-{VW6;Sd`&>NS@3?6Nm_%*<+Vp_NC$@}Xh?_L!aT;9M$gydR!(-7OEx-5tMp7Xywn6?4u9Gq)Q)ON_7dpoH^~ft6Z92 z?5A(H16K*mdNI-M;z^~trhB`__lkxebvP1o#9`0ggcvRK*nvl#Vbwc)9dBc4_i+S2o5_kvvw8TB-i!pfduU@TfXH@KYC2j zZ*_uTht$KvN^>(D-4&G1UH!4y>HC;Nu6@%++fLJqGP3R>?|hB880?*OD(~Au-!UG$ zjCRj3sBk?yNh^7$gj`(>{}c1x(F1~Hitm#^**32DPj+Li?p;VRG*x5hXmu^9dbf8? zmUmh0!}{6GvbmWr4vN?Bh!yMRWrha%=Y%%MtDb6ZiW59H#aCiN*4MrFUG{yqld7t@ z)p^NIYU2!v{_6U#vl2R;~X2~1O~3o}fg?`vtV*5-TUq+1?jO8k-mh9d`McV~Z*IzQ78?%aB<-D@x8th8 zZ582}4?b&JH$4`wl0RIXxM#y6W1njW*Hzm+^6}bKx_pAx28$ReTeZ>x=G!_$jfCjZ z>rUl~Jp{f7qhTY+wL)D!jSRcDJIZRcM4iRYkZ zZfQ9&ztv}kKX19M?c%pa30*xmEvIhGpZcJ1Utx7+^T&uN&oj9awK_hE4QX`4#!1Dc zs|r8Z@y>taIW~f^%&Mi`Ep6+YzDmpS6{B9$R_s|3R{nYJsEIz>0{zwDz8}UX=H-1@ z89dpB`J~@>ublMjWzCM#)4wU_md)S)_`x{0kf(cGQh zXfWnYQ$(y?)30am6J$M}HwcWk3?4MzFe7Sn(1P79i7h+=tMOi(KK@CeFIL{-6_l;uPoaciw z`^K_u?cwKjD*7bYHs|@74}EfMoAW#i(XYt1yM~`H5%1Sv+fBpIduaC0V%tH(&tEL- zH)h+M=OLW?7qe~7^9hIhSF-K!;pYWn`q#5GA4F&?)Hm84d?La8o=JZ#(4Mef+;Nkwm;DI={&FRle z9!O)`)x-U3w+FJ=HmASq^*{mJ=JY@P8o152(}w$l#tv4pZLZ<|8Rfwmw*6$dzhvQH zJ=-2V+<##=*vPg&5BDeRA8cjYmBa1(k%L`qTXDF({n}t3+os|inJ}tq5D%>nf2b__ zX^!|)dd8mVA9J@4w-&K5t!!Ix*uVQs>tfp+e~XLW$F@2Ck2D5XP(Mr? z_DA&Squ4gbKd_*W9a=tozurlgVB4Jg=W)6m+g2LBpT%QvKea#3{pBWIgKf{r4U9T3 zs$TPgK8tN9i@(nbc{$<2ce*j#e(v!)Z+mh0X931ywyh%~NNm7{`@h-L zm_2IwKMQ($WxRXC4du29XWM%0`QOX<;dN(8%d1MynkB8JI*qdp_5NjQ8;=~%Z=FH1NMv&wjXTa5%wn7)EHCF{;8BLGr|^tJ#zn}|7P#Tx+9lU zV;M)?Ih(?Y~QA{z3oimi>_Y=&GeG#W+1BsXPgwPg+b=R66ihgvo)w~4xgxsqfu`b|AkxTURX8|)rgv*w_ za^AV!v|8XHq5or|`f0U=a9!`B1O$9QmPM=ZWa}{aJ@sW?I1# zMP676-Gm7$sd9{ChyKkD`QHY7 z)u01B1fAd!_yQh-uYl^P{{|>;g7OGz!FTW!P@Uvd$A33?4t{|bpa;AJRJlQU9hB$L z0Q$izz^+IjelXw=qX#i84p80%1_@%e7YxP(7zT+^rYJuERo+r%ELE;jWgS)i2?8M? z48{P;+YkZN`)M$11Th{=0AfHKP~L|mmKU@o9MnR&nj%m)jA zDOd!kvfm6W0ZYL$upF!a=3phT0IL918Mg#$z*?{ltOr&A6A%&Bzy@ptw!jY90|&4P zYzA9^BX9!FU@O=Lwu2pDC!oeRQuj1h;0D~mZb04FJONcv@Ph6E-e51-2lj)5z#L`t zA*e6#1BZb>pep_W&?Dd|SOxnSl$!70I5ZFhfd&u^*5LP(&~?yL&=7DM*uV~j+Cjsi zn1hA=KBq0v2-uMz3haP=9_j*(hPp#zpt0a0xCG)rJV*dOSe^(?0?EJ^b{aGtTn0yA zUx8+TOhC;qb`^RYdJUQdu7hll148h7F7yT{0EOT-C zJLmvkz&G%9Si7L#!4D7yeu84q4SsReW05KpAB!DEC2&8~CxPbGJ zfyx3oFd0k%Q-M5~2BrfApa_(JGME8WfGSV}luthsI4GhVfX)K?;4Jn(8@dU<&w(0& zxxfd%8$*wRc_0v&fcannSO`qPBCr^k0m|Q53U=b2v$BfUDp%xCSuw6p;nOz;zG~vcVaU10p~!hy*vlS&#?L zfqW1J3cz_#2rhsk5Dkh!47dqm!7XqJl!7>L8^nVX|053s3XaKLkYw!lV1@FLn&({J;u~1{;6?um*xa1qcCEAPm&N7@!Wu0u3Mn zG{HEa1w?^17!P#71fUDVz)Tzfz4nG za0E`k8EggHz;>_$>;$`j3vdN)z#Z%c9>5cLfjz(*>;?P4esBQzfP>%|I1U2A2@nK= z!AWomgn-i^6oi3ra0WzxNN^UM15w~SxB#L-42T66!6gs};z0sP1W6znq<~bA2GYT0 za0O(5OmG!k16klY$Obte7u*1OARiQfLQn*X!A(#NZh;a|3T}ftpbXpv<=`H;4=O+< zcmS%vL+}VZ22Vf@s0B|!9e4(wgBRc>s0R(;6?hHafVbcscn=yu6Zil=f={3sw1Cf` z6|{kN&;dHZ7w{E)16|-d_yK-`Ztx5AfL_oC`oRDg1OzG$nEr&oL>vU>J;9tA_z7?U zZa~fA!2@^!AHZ~91V0!J1b`qA0>WSn7z;$eI3Nnfg9$(khyw{A2_^z5APpt~86XSf zz+^B5Oa<~_8ki0gfFe)=%3uai1sj1Hum$SC4rl;-pa~p+7T5%|!DgTXwg6q=2xbB& zpa-16EU*>mgKc0o*bWT94qym&f;nIpFaj=ME}-g=#=s5C1Ma{C><06J2Uq|+!9w5# zOu-(o2zZ0VU@tHO`@j;gA1nn2z%t+imV<*}1vmuEfiGAI{D1{G3|0Yu5F>~O@Qu(CPF=+Nl;H{GSmy20^I{mg?dBNpnIX|(0$O$(EZRW z&;!s6s1Gy~dJuXQdI)+A>I=<+`a!Ql4@0w|{?Hs~05lhR1bPE{6q*M;2F-^ahZaBs zp@q;B&?0CMv=|x;y$L-Dy#+l5ErEtWOQENsx1piXJJ2v_88jSv7kUO-4vm1`gGNH{ zL(f7hpy!~K&?x8w=y_-r^a8XR8V!92je$Ob#zG%MFG8O{FF|XdanM?5JoG6v0a^!5 zgg%2NL7zjDp)a5*(3j9uXgxFy+5k<5zJgwczJ^|bzJX>y-$FB?@1R$q@1fVAjnFJ; z6ZAUt12h}@5t;-21kHsuLvKJ^pn1^G(0phsv;f)$Erhm1i=Z9QVrVC{8+-x3z*o=% zzJcEV!`^woSy9~$o7sD*B8t5W21_jK^kPNfASfaNiv2PbR<__45fkHrz1OI*1?;`| z-h1y|G#X7aCZ?Oj?>RH~?%wSM(EQ%t=lebxSoZ$!nKNfj`B(T0{2TrX|ACV>;Q7Lt z@Csx2EUvF|J)7%mT+iY9I@fc#zQOh9T;Jq+9@n?Hp3n7dt`~59hwCr6zRUGOuJ3WZ zi0k`YFXs9I*Gss5$n{dLA94L9*N?d_=DL*Y60Vo;8Q;QAM?cXItJ*Som>jqBZ9|IYOuuK(bAFV}x^y^rg^xIW1B zKU^Q;`d_XObN!C1RJbWn0XZN!`hbBj2nNFt7z)E+B^VAX!w6UfR)vwU8mta$z?!fY ztPShHy09Ls4;#RSun}wwqhK^_0zZN=uqkW?o5L3HW7rb5f~{d2sDy1n1r2mCzyu3y zaKHr*wu9|q2iOsIf}LS3>;k*OI2aECCcthm5hlUzum?uGe)IuH1gTtU68lVxH;Bc4^ zN5BF&5`F?dg`?nSa5OZ-LRbXHz_D-~91kbJiEt8}45z@Ua2lKrXTX_o7Mu;|z`5{q zxD-BScrE6-1TKTi;R?7Cu7a!K8n_m&gX`f2xDjrGo8cC?6>fvu;SRVH?t;7F9=I3o zgZtqDcn}_fhv5-;6dr@e;R$#Wo`R?08F&_+gXiG|coANLm*Ew76<&ka;SG2b-h#K` z9e5YsgZJSB_z*sVk6|f%0-wUK;4}Cfehpv1m+%|-E&L9C4}XBK;E(V%{0aUH-@sqs zukbhcJNyIw3IBp`;otBd_%D11LpJ6iz)(|of*-?Z*b+8@t>8znHH?96U{k1s&0t&D99&Sr0}ZwV z9kvGpb^sH01PgWo8+HZl1iQfQuq*5V<6tt3hdn{S{;)S32-D$U*ar$Q6Xw7y zr~)4jg#hM42-Ofl4a86j3Dm)CI1!rRBv=S1!y-5Zj)7C*SU3%igVW)7I0H_A3*b!n z1)K#J!r5>UoC6obxo`>m94>|P;FoYdTmy^YT37uY52{*x2a5G#D z_rNW1FWd_E!92Jh4uc2ac6bKr;aO;a=b#auhbDLd9*1|}6?hk3h42%~fPcYH;afNg{te&2W}|5n`k^hk&gE6uitFcGb*@uj z7R&=54ub&dA%q5qpb=tdf&>nS*)SjGz!6Xd3*b;V66V5Bpc;M(HEAjDVeB71$Y8g|RRac7fGkS6Cg! z!5T0g)&v1-!30q z5ln|MFatJ)ePA=#7dD5Pum$V~GJo{`uq7M-Tfu>_H5>%nz`;-nhrqT_fLY)}0CUlE zhg^@^knt?9{CzgpIZy?M!d$3^8rYkEujP6e*E+7`;2$uL>ol(QTpOSX4u_w?&iwpn zu4B12b6p6F;21a-j)UXj1UL~+f|KDCI2BHV)8Py_6V8IO;T$*@eh%ls`EUXJ0xpD$ z;9|H0E`?vhVpsx~!R2rTTnSgf)o=}53)jK*a0A>3H^I$t3)~8~!R>Gd+zEHV-Ea@w z3-`hO@Blmr55dFm2s{dp!Q=1*JPA+1)9?&D3(vvx@B+LDFTu<33cL!h!Rzn_ya{i? z+wcy&3-7`E@Bw@XAHm146h47Z;aBiG_&xjqzJfo)H!$`p-V4|j#=&?HFadUhi7*Lv zhdp32>;+R{Z!Dn4;d&|8Te+UV^){}TaJ`-Dbo3o}aJ>`mg1g}!xEJn& z`{4n25FUbu;SqQg9)ri>2{;#?ghj}GitFk0+tXa1foI`4cpf7D{ROTs!b|WnyaKPn zYw$X}0dK-v@HV^y@4|cVK70TdDV!y^{tZTd$GC${U|0ANjDs;S9ySF5o52Lw9Cm{( zU?Th&Cc&1lJ8T7ez}7Gswt+pN5~jemuotK>6*SlzbeIMPOa~KYfCc-24f}!vGr@)Z zz=Qo^J2(Iapc?$MG1$&-?Jy5}(r(h3Mu;(e*0~PFf6r2GUMuG>c!FI4ZY!7R| z4zMQd2y4Mkur}-r)DOa*r(h3Mu;(e*0~PFf3idz+d!B+lP{E$3U=LKV=PB3&73_Hm z_CN)Do`OA4!JemJ4^*({DcA!Q?0E|IKm~iAf;~{do~K|BRIukM*aH>pc?$MG1$&-? zy-L9*q+qjBunQ^JtrToS3brc+`;dbDO2I~?V8c?d6DioS6l_Hbwk!pEk%B!-!Dggj z(^9Y-DcH3XY)1;VEd~3Lf_+QDhNNKQQm`W_*tryJNeZ?u1$&Z$y-UHSq+st-uq!Fp zycBFp3U)6A`;vm~OTosZVE6l_fjb}$8dlY%Ww!RDl34^yx^DcHmmY)=Yy zF$MdRf^AH}2BlyhQ?NrR*vJ%YQ3^IT1$&f&y-dL-rC>8tuuCb}%@k}?3br!^`;>xB zMHz5D90nIaJ^TV1;6iAGi=YWEhQr|!m=Bl25%5b`0E^*BSOPzV%i$=v0)7Tp!qIRQ zG{e=f5Uznma4j4I*TJ!HJsbx&!0~V+oB%h$iEuNV1h>G+a4Vb&x5H_02b>Of!WnQE zoC$ZsS#S@W4fn#ia6kMU9)RZIJUdtjkHR8&432@v;aGSAj)N!Rcz6mXaSK(}U4bFkr;aqqFehzQKdGHpT4{yT- z@DBU}-h~U{J-7(ohl}9@xCB0gOW`BcsTnV4URq$)L z8oq#Q;7hm`egoIRZ{d3Q9ozuFha2G!a1(q5H^U#{7Wf)&g+IY<@MoC1GGi9@hQnYQ z)WdXWfEmyT`#=-y3x~r@m=F8G5wJfjfCJ!2I1qjU2f)<)K z9-fCA;03r5UWA+ACAb-0hFjnjxD{T7+u$|09bSh!;0?GF-h{j0Ew~%rhI`;0xEJ1q z``|seAKr%t-~)INK7@zhBX}4-hOxEu8SDZ+>PqhT@J4@=+yxC|bI%i$ro0v?7d;SsnB9)+vnF}MaE zhil;pxDK9#>)|Q50iK2%;TgCIo`sv?Ik*L$hg;zVxD8%}+u}2kHE`2`KO*sDLq$gH0h1o54WX90tJ_Fc^LeLtsl73R}T2*cw)XZD2T5!pg8M zi~to@0S#6K9Y%rytAPotg9U4V4Qqk}Yk>=Eg9q!tcCapN59`4Wus-Yv8^BJmA?yqr z!C2TBc7ai_D~yJ5a0`3agIAxWthV;jQ4elBYV=RH9<}X2b(XQ_;-%(4u9#&HziP3) z(YH(O0smdP>wgq~T=U@3yD7sy*=@|qOD0}FV)X9Yt}}X%?brR}fNwVP4^+)1;YR-G z`0GQL#A9nlCwtX>l6-x$Kl_pgmdqLOvR^g()laGhyyqV}VCj;%H+?#~=69csu6yOn zC3W9^;~&PXdDz#=&h-P(4IYyp#kw|&b}b&T+S;Qw9zJT;#(&&;)>d1pOSh`KYVp<; z*Noa`qi;u54*2gZ`_k_g+y7BUJIXNM8ME?e_xce_JleF~Hye%KW30Jk`bPdIGrm65 z--ovBbIAk#zO-Xz{OXdK1Ku0GAMMzmb{zQo&ptWml`s8+M-BMukgt`a3)0tN@#?b% zHE*zZ@LC&>8gkc|u|o%Jy?C{;S1nyly?)jj1HK)#*68n+t~DUa8Uk4)}PNP1L2MHy!YsPc~KmG|5U%(04gw1JCS-bgDa>5 zQ!#AdN);pWt5WT9wSlW|um)vE*REK1nF(rz{=#NyV>~FY;ejd^PIp{68xGHSk+y_~zU1G-a(R%bcAZiEZw-{osG$m&aU+4^7YuJ!=wijM595Pg0YvAbI`UBVB%+2pHaNGPaDn5-J zwRwK{P&>EX!1V^_hYy}Q$)UX9;L0Jx25pwxa7ab-E7Mk`VA1HxAzQAs-ew!DvG&j@ zeA)Pw*Bv};(8M8|4{cg$*PXT;wEe(ggZ8Ez(5ktCEA2m|V#8pgi9?1pUvg?*>95(!v;+lwqb6!NltDA zU+f`!>-@0hzpXoX``m{6D68lzA9GRCv{LgECr=JnK2}|Itzlgj;8o9<)%>5_!aD}7y~@Z{b{pEbL-XH;H4K@z=7b~H7{12vnM2oY zK4IZ*x#MP^M^~f$N(U3rB37kB09%tobH)!>^BLCqH)^KNb= z1$>*|8^l3vmEn1po8Mx{*7=GNLpNWz-;fO{@Vk9}BOX)pw|E~>GKpeh6_hj@Japu+ z4OUur__`~vI%4=L`BihPt+x8mwUo7~CBANMy`k$X8&qt()~MW872B*-si=9aLRYS> zxGsPFkbhNtJMh1`?}pxZ#DY^!yG)(A-zle_xxt4cR@r0nxBsr(cISf*E_{0IX{Voc z_O-X(cmI=5J^S28pMCz_fPDH&INNQ%!M4Zed*-2Od(0;?vH$mQOtS?8l#d{`JZu#iS^1KK8Z;9(wrA zx4-_=F~^;9#Z?bI{N&RwzxV#c3m$y&sh3`!G-a=u`yWy`>GU&hz3c9WAARcSw@0qE z_JIfe?eG8nuDRy0k3Jr;ac$iO8y1fG*-bZp^zokeKd{!?8*Mycw<&wcm*!_j-}&Sl zZ+`IgpZ;3ka7JU(W|iAsee>N9KmGFCA740N?D-d{XKeh^cT@H{;K0E{R#|oPZNK_s zZJo2@&bx}~vzwlI_LW!PdH0L|em7uLVUtB4=NFA1vVMNh$b~nIXuf{nsKE<2$gN8i zt8Mcp70r~vg9eWrHg%QN2Jb_usSSovz=~RCRPV|Sr()YmgOm|#4xBQ0{lPP-9kyh06cuV{DzwrM%CmG-L(0cfs1a*tv6`Ve{%Z|UTf$&L*E{~i>POWj=?4+YfB7OZ?qSCrNAFnU zHxez{kQQREqD>R|cj-I$Bd$v2>nau+ese-yJ-?XNnfL2wH`T-h0M|7^k_DQfD=dEY zLOv_w-AaEykH7EE-)kElyR)IDp@Bd0-`6{5*D+(W>;Bnw>>1hh#K*Jiq5nv)&Bs>m zGJyYHSuISj$G&}jdW~OvQC>$C7FX=bm5dmUyk2(hdNo|fetxlf8P_v6dg6h%xSE^& zb?)$S1DcmQ3m)HP+TEeB6L%rp&H+^!jlFc06m=XZw9JZtSUd90h$1UxSN$hUGCUPj9qpT2oW@JF3K%{}lct<5?Id)p0ax*0i{xsk*U!oL%jxSv>tx zoZa6y&f*r2;!9@m`BD6L2Tp%?!97sH&m@Ckz%GCJCts%c>F^-Dz`W^pmxdR)W zXFnR0uk~x<0c)-?$FC|7Vi?8^4TYMjBP!?n4K>Q5j|M$IY`|Ll{D=jG*>Pimd>~0& zUkDa7#tlmG=ak89^Ye<;T7REE&5P?B7vzdRlfPT~S$@6j_Xu1r%%Pu=TjS3w5IETs z4=nzh{LZp}o99>6Hw^0h-_p+wF8-VRp7x&`vQhSZ_3^xFKa9)&J^k#^4NL!CQ+IgB ze;!u6g|ND=AucpDGBj#dS~dHKY-U5F-`LbJe9f|-%<=09@Q(aOf92wj!@62t(5BMo zM-+c7o7>R(+pBDn36Vc;tP1BA=)bW7$otzJBYs#XR1ykr>p1Y-xN%;~J$qOut6n`kKWq zj^e|bVtFT7yVJj{Rs6TOw&TC8UHrHDxS_6^FDU=qI>jc`$Fr*%8W{-X|G4fdWp~;T z*GB6N%YG0Mdb)o3KgxG#gW?b7G&atoU#hD&T(@+4wNatEjvp0h!o0fLhIpgm*UDcj zBz5(1)$H0rDW-Pgq1orE>t>JIt8l;+lQ*JDGpefs-h|q?o+&-jb@{_WL#;orVNRVK z+VSC4@qA5F3GI#QYUDekIdkI3HdW0?EIrU8Hwnb(Z3+eAbL;AnQRrMmA+9~Vs=lsP zirB>H{jWDv9T_*s5B*5(`9q9lMprsU_{5kkd;Ua1d+Vzj<4w2f`FH8Run-?n)wtQn zg1pO`^m!D*#=823&G(>7H8tyE`EPb(y-yipO`FJr8oY;!uK>WB5!io*cMiau+F!5a7 ziW5gQqgXY>mXmvPn_ipSY>}n~o)O!s;l_rZ_?{-V>a{dh|MtBW=>bE~2?9r!7&k1j z^`5Y@d7ov!+hL&0>)r@NkO zC8Dx-^4gD&l04P$Z0f5!iSPPeNec8R11iG>AI=UITeq?B&YYVk^ z0?QiMR-kXi$cjDJh^QZMRi=y zZ5vDkPH4xm8ET$t2EyvI$l{a9whVGK8afr}L&M_X=(-UJyLTeH=*b2|`d%0Y3>_

y4Yb;TV>qN4l#UNo)wW&mueAxE>j%=2?*$#)0jt!t0&LE*`kNaWhV| z#L)u=I=hR+6WjGuWb2JvE^-)~ny>4z&E&IvpGCI6aZmFCOLrWmNz=DfUFMHQUfVFL3=ZaE!>b#ZG;e+2+=z%+OIS z-8NL!_8ng{bg^@PWwzcsgU5?4mT4#U3{wxq*xt$PFp$dbJ@8qhRmZShmG{#y#V&o8 z+2P)ugn7+j9yWr|wlq`h+FzNi_ip;38>mqnt5IlbdMw6G@6A}U%bt7Nv6Ma}D|CEU z_f6ICZ9lQZ_}&SvkF(=Urz?gL1v>Z5B%(R7?OQ_hPH5Rv>e$jK3LTS8kQu14ZaR^e zFrzoO?{wDE)=pIu)il&N@%7M%yg(DX^+#%(=g-i$j6`J?2{ccOg1`vH#C>|xRyL;P zz1p@95F6;B=6LJ_4Kq~L&=HgN?TuKkY&I!hH?spbat+^g*ktlv#U|5!Aa?Ja+_I6? zsa!wQA~y~UJ>Yl}B)-_AcXG?dL8o#ZHwd&ywpi1R*t5jsnZ4)-L+#KH#c|Nm5t?BJ zx@~K&nwXYqs$$PR3vM$MS_G?!8|sE@CSIV&dZLLby%XF;4>z<3b}f1UH3wtZS0m3A zd-YRr`>t?9w$7^Sc^(_+*!RWMJ`3)mD|~~qM_^jI7kCcG*1dZtxQjv2+7-I4hk@rO zc4)bp=7wTgKLxk%3d0FvBuC8SZm6-~PVcke&bz|10xQvU%Zoxah`dnD*snL^SSuUH zZRd!VVc~1KYCD>5Xqsc$?6CLgo#YPVt=Jc>y&RaK>o_W#FVogE&k_6fS#q1sXqC*X zVSAeGS}{{SYm%7RU&$RFAiG&3&@Iz9T-yp_v0v{bcNlUVdc(0yUDJZdGUCwC{YdQJ zXUSdkhUF%1WCykx1*#thzBr)2lH2!2XhnA9*vy_DhY?Q|2kzgSQP;^CwnKL$M(oJKE zdC4N%2m_j^i$i*pAlp?YA#~+Q!>|G~Fg4YTMPXbIZLs7El`4laMJ~;-bvxqKWFiUe zkf~M7>Xnc(MJ_@T$L3~O2I&^WsKW12h`g7!LUfwpn30iqF=uf*5W)B!ZpY44!r2{3 z{U%Le(_~mN&pTmS=fEt&9;L`mDOV*UMc1Ox^H^^*kN>wF5%nmA2a6t4D&8`UGg4eN z;`D-Q!E!^iZkFH?b*i3e86-}ocyB8v4t>jFx@YrhczUdfWI_*O-13fTX^!qCR&2*Q zGrVToULLsG8 zM>)kEY1@J0F#5DO3OtrAH=;dhi4BgzOpmnl3B zV>b+4-xakzit5x@fv8B0BQp-r!)bvXXtAj4LZe(7+PYCT_eir+p$03QKRZSkS`OC; zV)Q%_(n3Sb>p}%9zxkYHY7?;hmg{s=B#wx64HF2Xa zZJ9{55tb?IV;`lSE{^Ep)Z8Mq!w^r~SXDc_Tx=4UXmLZ`iA>9pCz%CZY~NeNrmJ?B z55HxTt;UQh&D3HKV~?eU;>a$xXf1+Ubun$bHXBu~n;pvwV%8r$bR3nP?DPdLA8;LpMvHLIWFyAOz-3>DcF`0^ zJKYO6rHwdkRcLXj_t@$BRuZYASxMgmtDW0}rltLtR8?bz){KdM)kO4W6SBXWEla*k z>MkvjUgDtQ_0jU^zD<{kg^GMgwpti7dBzxaU3Fn)B|XMCudJM&UdJ@)>KMNvy@Z&Y zD=U+l#>&d^>?W%H1!H!bF~;Br$o7LU@m1Y69hY|)9rq%o3E6HYJNqc2D=XQB(Ae7a z3#P|@J+4Ca9lxdBua`twk0{05nZ-=9}im>Ww=q=sH1iB%_z^q3=zCXQFq*{v^4)U0%fVDRQ? zw8%9yCorrqvP{&YmN-F4m27>IYqyL+{1xW=^^tsHx@Kts8!B6~IXg$=-3w7!i3{1hCXPl;d|5ii*q|_; z_ey`foBXMKyb_uca&t_vzALo@N$87{l$e${+2H9QyX<94$II z*r=FKRtj>E+UX$e>fUCgE$!k3Y&TGy&~W%mjIA`hK-MEX(QetUB~t@y*+1UezA)f@yg6WDkJ{;KrdTb+~9c95igYI88a2 zj?wxu$aJH0tBu z-d6epzDu=LtYj)W6ir7>7l^Z$_bZ~xsm$O=kzkBP*CNhQezrV2Om_=onw(%v`akmY z-&D{1;6(BVk(_N@8!rOi(IO*ou?yPb-0!&oksYbLYPKI6?Ayao{9N(-+yhQ&fN&V9 zXH3Qg>PXCH(wOhdzG!N!s;;b@>en~K(X=L9 z5Nl#l8_lb$s%VB^v*|$&2{I`lazjP+f)(reW zbl-?l`Z9GIwRQ6m6=C{|?I>{E#L}_Ui;EO_Y;U04Bmnb{7NMw3+iknK!oH4&{roW&xftzxxCuPI zvSlFM|7Bm=VbfN+cPg_%P1D7tK+hKgO_~)eTIBCw(OcC*VH%?IEE^gEblot2@*^74G)Jgj>?H6 zE>ljCdp$2(fIGYxZFX1_%a1eVJFPam`qU4T6(#e*l)b2H_@Snf+8l;uhnh)*M?!ia zb5Vzg%az?^Yv%Y3bAtN1`LzZv-MTxGzwYVJlaFzaG2_PU3)5j2DO_p>U?eN|BX#-buevvbM2K|I=+o&Y`7daJ>A7jd8KmJ_tPY!)Ze8I zSW)djKgB49G4yI^bV97Cy0}VVVgEk*$!u*@TT56v4medOIJX&^Z6=N>t|pZ7duWtR zn{d#k1?+Y$3=?6Z<7dXsT3n;-KnpZAn+fHSK*!mVQMgMeH)zFEZ|eCkKhBQKR5@zn zVWX;^Ev{AOmVI1Kqv}nyr=_!uOF<@r0_uW8@t((Zp3RtiRLH|Y z7PYyfS$5!w%IfOFYbxCW@m$UM(E;L{#(QKxRLivwn=OwllC90K34?E6! z-E`iK%BkrakgXGArcD?_z)sqr_U-QU>2=riHXo3!$3+~W*#MB?DzwxM*@L1f!7zfh zK)1zBee8UWQNCs1%c)~{Aa+39{6pPPLwp!FWY7#eRXoGNSnk`VxTTLbJ1DWaMccPV<=6j7YuG&QLwY(;8d<+R|$ zn(5=S6sxhy5@?INi;MF2@v6DyuNwQSrcgG?c6|#6a*w?yPQ{uh?ok@KkCwgg@;xnd zJ!9Y7i|#{9%>Xt%_Q;lW0ucB8;N6F>ypKUBMC)%D=wQYDsaCnQ`uCbn5f2Rd?zIYO;v;|7XOnt5Es z?&67u`Xn!FUVH>mW!ZY_vN^Xgm4~>0^9HHn;Y^2I+>La9ySm=yb_+x!)+g2J!4fAR zJQT2JM#Lwmo*^FTM>n?a8%zvhYGfse?YpRmRPm^inOj>I&TjT0#a$1Zo!Mo35gv%< zI%q5Kqr~+;5|1faBu}qoTE)FhSzqG9!^Xt6F|}DzbtWFqG_k$w%i_=}$!pV@_*U?Y zg4joGjWSa_p&Z(AO6+RS=gf=ax!iZE;w?yxEaND`9RWXWJk&#+!V- zZgFJfTDIE=a8t&LfZ5f<$cE$VQ+<_FT;w@OxK0o{9N%fNDxO~MI)!JHe^U8VY^C~x@)cEytW0Xfz=TK|9u4X*;fBC#RSW>c4 zoEcu^388G`Je@6mdP>sD7IeN}3z%)O+@W-gIc5{_*X8l(h4eJ(X1k8o7bZJi>`O6R z8ke1HdNr8ZenJi?y$L4=%*?4I+U7X84&pn3Tc_&a6((N%q4tkW23v?ZCVDy#W09si zYT#(Lcg0xiSX5Rple4tt+bEO8 zE2$;EgL%l5%4m&vpGamKNYiQ*>YJ+N-0iy_5#157z{G>#byXaXj^>#z{*ER-t(xh0 zj;Fn0##bm3E!2w06fcr7!OcOyr!i7pPhbIS8)Ph9LmoYr0EDx zyn1N^2m_67=1t|0iQ9%FURQSQx&>Zld#Z}EJx)mo`+hthu-T~irg0PyZz$V!EkvUf zeN8woZ3`_hLu_Nd8yJCuJE7r=H-9L*-&@(ir_&DX%{9qhYMvj8_qvsl@To?b2SNH3;1W?KwHpSZ#IyomO_nY@GR6Dd%BYF! zXSX(^-mj7EknVQ`0%~r+@i%cj@j$*~#7!G%aqViz!D$Ks>3DcSjlFEcS5)a&|m`5sO}zgA!CSIaeJ zs$9WJ)(3peM{yf~n zqioVE%EJKKR4=D>>FUK<6*ph977&glKI>LgSWO$Mk_AlZsgbk%_Hh1;96clqH!z4L z)Wzq@?p?l6x>H&l>KbvUDw$&obLtw>^P-2pml`Cj&6e)s=`DWUxf@ehY0$AOZP@LLlV0j}Mw zPvQ@qkM4GDvTOVW0SR=P8s=4nRdr2iqln?f=Um3UV0O>~@zoFHE%?XIUxaqI;IO-B z>4}ncR@#$^Su+evwRMis;_K8Apod4Kt5Z|IhF$QsoF7Ls$bA;`j_R2rv-mgOvIO^u zfPi^dV*!Xw@h2s9uWmoPwQXqU!fk1+pD()|mpisYv1h8xc^JUNpHpY99-BNp^+0Ot zMyDB7wGH!91JazP+PO$q4GUWu_D3J1mQ2R*jZ)Xyl-0J)1|b}cQcNJT`>>8Trk#+g zXykGx2cDL&2e#1a#3*^$@!KPmFzAJOfVskKN+SHO4WQ2AS z2NKnB5*`NH$!uGQ+X3C{nU);gw$93-{#~iz4jiv&<1QM_D=X{$`6QY1=g%PTT$~-a zi{|ojYgKmKuA7GwuDo$h-TZI5QIUV;IirpuA~9emsU@L7%<(^zN7{8+M^BU$*=6k< z7R&mKB~W_CIYvO322()SefH1PGV5eoiWWzMzV%s`UUP9AP$yMD7=?-WcEx@l-xY_du9}v|iHKmBf2W&`GKUR@p00H|^fVI>iMXEJZS}G2 zcLo>{lD>wr%_0onHSxgv&$5YQ=Si@#WjQLIeOS5vyG-H?PP#5hchE*h=mItI-Li=b zU2;>&xPydPPHwijLV9)ex9yg zw21}iMlo2QY6Uy7sG}pLF>XNgx*@~Y#jzC!l~44R+hLyXFiGRGoKz8YF}1pBc2zCT zO#ChVKF)xg*=!q6Ef<%+m;k-wD&(@-vW}1hsJ(YyX{K%OE#KO=m6g?X7)nulamK|-0XRCVIKW3ry1m*Y z&lIOFyUfVMvP%jYJ3*y@xmTQ4A&>ukX{SvKNoLDUs~&OZQH1X!CqZfBJbf92vFX8D zg;tkse!!b2&R7Ovo{3=?FITqlQS4i;ICB|<`6|N-XI59&LQ*c-;;dy6W@BMyi|!fZ zBf+gpoZWX}^@N~SNi`m4xlp&*Uz15kW6vPYSq5Qb6ZPp2qsyyD^t#APn$q(7`9(K9)ANkD4_90Oys7R@bF%0x5+ z*=E1!3tsk{m6hqQR#wX2t*o5F|B`}Q9t?W?-I4`wPkiFWwYcB*`eT#1hjb+y*ZZ#$ z5rJINtX6zu*e?=M)R37SQM@{??qSU*NkW3v{k#hNyrrjR79^G}wNedRSxLO{JpRD> zCY_rw0?S*T#C=Ri+tk6YJo{MczQl**-+Z#oR#r|b{z-nSq{THHOarl(MC2$@$D?jGOSKsCQ*zdcNq0qZ0+LlQ;p(xtpdxeq?GO?w zKOp6!#Fipc<#l=E%KNx?okdmz!US+@z?Ui1$+_ENtnJWQzxbc%EX#~6=@KlnO?V!A z$O|i)+Z=uEj+@BxXI6Z!*#=Mg>r_^zzQR0lnV`J%fOsqsF(y_-7^+A32q*4ty5^z^ zX$2|WN$2P>JE97g4KD9`GYcdaG z_p>bV%N6o9o?J6xo(d{Jovo=@yh6SP+W=c~$0v%(?ep(NX!w+;3VQ;Epb&vk6B`nl;$>QA2~zLBe`wHN6FQ3 zdu`9)=Ef&{Qq?UcPLl>3q(4kqy)Ce*w0_$Js*UpgPOw@arS6_9woDa!M z;qoI{E<##v49wto@DKukUeo6odsBtX&S5k(Mn=N|%wCOi6ux**1>u8{qpL02dH zM=qGo`$S@?mRCwsi-}x_XF|G%#Yv6H*c_0738!Fjb9xdoGhO?P@-1Zh7f>7_O;?$x zUonG;&iU9#P?;L0iuZ{xZdn#-w2T*+164$bvY&+F)@6}~zXj(IssoU1h+J5)xNTXa zVJ=7r;b8&A?>WIK_x5FxMjkDb^tMruB-leWf<=~18u{jZ0+&b&fh|ZAcP@)GGD|p? zud7%M9WpEi;;wY5$#kb|{@hae6njGLUKFL_?WML#9e-kYktZO*jZDKW$-r6~h`al) z*R5ywA+=OhWox}$J`j&DqDg%V6onQ#F;s>3RBX??k!`p2`cSi`70+tLJ+hbTc^fj< zdTKhTgz}q#VLLL{qPVwW4xe_i_|tF>wOHy{qj)D}_&jP}{xpL>rSH;_>3N{+_?Mlq zvM}ckseN#uBZ)g+;_T~io=n7jsm9JNnE~5YVyf))mdn1Yb(JS4PceG8UGJp%n~I)g zBu!;NNS0E$=t${tOP_lW>^r{HD7- z0-#r>cU5+Dnn!k$2P#&=E}iBVV+MM#Lb}Aa_+shVX}fio(U@5bbbP|dk(YUI^4!T7 zi)x*LncyKywGZ)7g-ond3`U<&K5J!H$2h$|lMUjqqcBY`kk`!UV18S%v<{gi`6Qgev$42dpa$1EQ0pBz64RGY-kI4$6k zswU#G^hN35wP3Rg$84ghX$FB(iaye>;6Q3)z$UsS!cX?`iUzuSMLJZn!L@aRG)5K? zt`>_XnU0>QNJD1Z`)?H6tP?$!_&nJ{4Z5=GLUsp)2h1l|j`Z9h9XAf4w#iC@*;Tc~ zlNFz)+uq`P(4|4C(9gYZmjc^csM>9W3x(3&w(ON}zxtMbMQ(oyb&>kT)W_c^r3L|% zc;gXu=nx6Pj-Q-jD)@5U+IETt`o^&!z*T>=ynL`PB zC$&YmWk5Sbh3x1IxSYyCQK3-e~?EZ2-_l>S(AP$bftK`>)S1t zXUb%p^HRSCsV8eAlgcpU!-{bxb}76;BI`FQdd!`kio@eU7V4x~o`X-e??(<9wR8*T zz&CpqAv=q0Xh{S)07=R15lO>Vfpd&_tD?uO=du;WETA%hx)bqn9|X1NSP?|n0oF>(CJly* zqkk-x_AIE=Ozd%j%Q(7m62egSk64^f`m2SONxV6aNc#f{AZi{U{9%gK=d8A69_@kP(Fx=Ir7lbTVU zgcJ1c6iM>Lm#J2!n^&*9Eb$l!@rjE`@UNTLNL2BgbQ9N|wC>WxlcI!(V2?!hDwTS0 zO8sp`uhP1=yn$GQiYcOsD?2(s#;QumOA|G<>5Jd>PGZZe*g>L4CeoN#E(*7DGACov z@4H$&Ti$}6GsVk_L{!2H4gBK+0+;@fZeF|T#$Kd~$8JZ4d@|dSo=TI21;6UI(Big+ zX(_=NEK9b@5}I-Vi9c2>hh*_KNd|Gf#i7Qcqy~%J*A>enTf8isO1%+{fME))MV9ze z#d1j(&k-mzNhmF2r>H>dia%E@r+o1g@vy0ZiN={VgbWeln~MIXb#I?AUgj;QG!EgI zY^kMR!e9C=x;PwK0+TeHlTnk9&Lbd!B`F>LTCx06#^Wt+Py-BCDH3*>y7*hg^2-_T z;a)~!c9KeBpQU=O_aT$(9LS>s6y zf}zFc&PjSK)s_G3gIzW7QPh#<*F@3HvCJ3$s@R1CSqCMXS&v3rGuvvkNpKxZmE6yY-G=;GVHE6ht8o%U)b#CLFn{&(tK(sK5vq*Z%I=neT<{o$C9gKz z&{zZTy=6S$k;5U)JU-V+rp;nUn56_s)p70YG$V$GnT{DTNer7Z6O_w4AtzH0cixXy z)PxgbC=KDIo{XsWiFMj0MWHnISmMO)K2IPeDVcexT2i}^!a~IfnG7E)^~WfheNvmx z%Z{_sek)!|Z-X>B;6q8VXww@ER?X;qw@OG`4fH$-^odE)A-8WQOdPSqGs zmVd;LQ^N_r;D?7bRn^Pai?bPqN1|QHlt>CvU7V7Wo>yk(eABHiw?oAOeDNvt7*MV= zLX##=?XM(iq~T^7kvx&(v+kMVwEhoP(iW1;S#>xU;$x^;;`IK@id+xp{TM|KMG|B- zfHQJ(f7(0aGozXJlh>CTo`#L*MA3t_lL38ZE@=@<`HW>%fp`5yXXRvwR@#0uji}Rh zl?4i93`Sh<%Aa65Pe%#0lx;T8vTzzFc3YgCEd)+^-39skl$)t4MsZ{bYNvOuMb3UV z`y%-pR+`3|@rYR=CbO_%%JR(wN_W#u=k&Vu=uikT3wU25pHeO2+}?iD#$yzHq=~|W z5_Uvw|2$Vsk1gjs<6S?X^KuL1*R8+|aq)e*`Jo;K%tW`j%uj5QxHur7WYRA8UuqWIm~frWZ6QA zCFu&@w%5A2Un^a`r5mf7~L*_CBu zl*jFMDHctVvFg%mp13y_f1y3L-a9& zzGmYd&E=Sf)z{ zn{%I~_vm#|v~We;W7~8>9d5Bd{&D3_V|Ye~q#weY!X?`W1r2ZcVeYf1;kvB%Ngfmj ze^Js3;~Zi8rnoitdGWhy2Z;_yr?4{SW7CMl9sQL=x&Vg& zP|_$->dR3>acBQkTRw#?Eh<9FOrVq$RK;C6m1mi$ws85g(voJy13iD|Xh4n(lduy~ z5qe~Lz**+*^cd4(y;)YZ)jhct?PPgxZpAuT-k005Fm1}=;~KR6o$>x|GGvb6R)Oy^{3wmkSh^s*9tC za=Uc%8N(`lh9Ix(V-0hNiy*UNQ!NH062}z|*B{QM<>53nJ1=I+oSyPGQ8F;AH!u$q z2IhleXqVJQbZ1297y(8e7l-^X6dR#DCwg=7NG=OZ&Dv2^oH#|VZ%xhCt14rc_F>Gu2D)ohj$t6{;flv{<>hW<51Yo&N*~fo+9|mcprM3g;7CPcKl}_0Dj13<(s#Gr$Gq&T$S}f3 zY<9?$*JkP<)txL$`D7c7Qt_K~E$a}H+4{3*vx{%Q%_%KpNa6~@u*hkNyR3<)r+BK< zuPpXwR^rQ%(GwBjX?RsRkuRQZ)19T8D;D+0e39i>W*+C*6Ud$GhJ+5e31xquNj2~7 zZc!_KxHcv`aTR&us5HXOO5F&luZVp~2!O!@rg%!gAkSf0wbE1@CIS1N`fN@*=Vnt_ zZBvb&ZNXX&%5r4Ltv*vpI_#G-TnaZk@Xr$XM~wslA7>G5HGT%<^wNcBhDbTs}4mlyGNIw1NP^lTesw{j>MN>+AUnO*TtZUqFuyql9b zFftXh*|4CNc)VTey9sU$aWf_LzDyJ#L%dz_Uhe-Vr7BBG<+G9+M8uj2I$Vb({Qcb7 zy-C`o|8FMkYN=enT1l*Kx;w0_?957#DOQ=ZBUTO@l0P3AO8P&N4C!~Z{OWD*J0CsGC{shFUh*b^W1|5!{aXw3dPh4~DE z)tR?G&V@{T#ne>gD@9AJdYDFckDE@gc-dhkGo5r1JI7K)l}1x#_sGeE=eRUG`ljzq z7xq~z&8Dfl`(lr(Z4{+s;ce_&t%h3LAp;Q#eBUB-ut`0@PnP#LV5G)B&n3wgdrr;- z;?oZMU#HkdS-F{dJI$M4UvxFabBGNQ309MkO*(E_ze<;w@)f@G&zI8aQH-3z8k$Z) zc~YdZ(fDjR8M)~??#T|R`AA+qp3dj}G5V@F(y|K5$TXJb$PvFzog8{&rEPEIc5Imw zkPK!xqJ^5@a-N~KCa(CzrepZ_=ua&DU-Yl36tE77r!fpSpSjgMmmu2}8R$?Yir@VJjqoU&#{c2gBm=MTVEcXU#8hXiX7Tvt{oj=*OE#6^qDxs& zQlZE#v9zq0f3#&XY$HQ%1|U5<%aZ|tj4}@=MZuELBT?xe+9(8y%9)ZEWILTlfJaV* z5s?HElnTO4?5mDGs8XfCHod-HX5bX^Qa)<3_GULDxB1yG(7sk5RI08Ky@~rix)rK` zmaUBFIr$7+VxDzblg<2NU%ZKkQ==djX+J4W9g;aseBCDxR)Qed7fCM^Trw;@ z5P!Lj$Ry(2A7Am>6~` zeuuIJ?QJbalz{v+#=eh+yc- zn0VC27WFXtzj?IC2V~;Gj=l%~7h4nm?$M@#*E2!7oRqzotQa_l|7V$ODjkOWe^9eBbTpQ8_5^(5b!2#LPKdEX>DppI@GW zGFwjCe@(V*^meC@a)vm5Y!W!Ykik_Ii}G1|gw{K;vpNp;tm(iAbs`*S zv`NpY~E_O$h2)~aUV}*#K?7VbI0q8W0)t7 z%lALfBnou$%Mho?TFWvdj_|554JSpFkh1E0AdnP#d1f)#!5=Wfe|2E5P3PzO~=N$?}SF%iTDq=a;CmGCkO`Kj%=|^RE}vGg6)kTPB{AHsL%P zTewsEe|H1}k(V4pj!)bGDI~>ddFcadcbq2ra$m)U^4v(2Cv)=2`i-P3JH7u6#X*;x zE2M(uAyDr|6=&pK`C6G6O#CD+6sH#h|2a*QGY*>}X}{&g=irE%huEKlEWjj#5NGDo zz@^SUYNsDX6Cq>34y{Z5gd<(!8?egbW=Q>AL!6aQ6B&2*c_;mRYO*MhzKnWqa^{gq zD={*nsN@!iN)gLz`n_qQ-z`8U)r6?IM2BlbquHq-kC=KXF+cz{9&s zGhA8alqi0ZUtXBLB%QXWP0z~qRMpPI9-0Y87Cyr z84aL&<@cAwRMjTyUVWSu-IbYqr2UDZN3B6wLz}Jt#rZTrOjiQ4H?R2UGl8V;lY&@A z7f+9f-v*^Jzq2S9h1a+{L?-eVEE>aruv;kh-}PE(i&@r`)kvoKG`WbtTT|*!!ih z8DCW?^c0!wA{eD8dzSR!F0+33Yr{Epa)9A)fTJWTQldpjsiKR^@~Lw~7tJ!KiCxC? zF;?h+dSnri^xO_>_2qr&oU92E9?T>(JY3F!B)p+`Gg(RSP9{jv5LcuNP5WV9dTwT$ zk(Bl|T9XDMISVaPTYUgzbS_=S%!nP3FA`Vg(?aiE+@d#qk4aMtlP66PG%RLnmm^A= z|D%ww(5v!ke#$OBtG9Uvv-?a)M4J_9LLN8CZUv`XbPYU3Ph8z+dA!dum3}t&2PXS# z89>b&$7-k z;)c8o-s)<^8ExL~vIkoxI(s4{*P#F~IjE&egNJ(M#(Zk)@2bse$M-yYBCS1$8)$@< zn|&PtXZT6n)MtsA^}a=7z*=jP^BMI|OnOaA+?-ESw{+FUY~Ryn;w~MXSY#p@$(>Fn zvcg}Cb2+CdHDO0AZs~q9k#CxuOd@HXVKZpisbWyv+K1+FQ?*%5+@dc+A>jR(E7vE? z=O$TX!h||=Oe~(bEx$`A&$wtf(b_!X^k$Jk@kA$L0Y+Epd7A2r+w)VqmSMEnS)?*3 z$dDBdY#Dka9g8Rn9hh>T_{X&$0>n;rW@+Ergqm%nG$d5gLjgd5=RJApo6-JNF3D}P=4Y)< z<+g1s@i}NMD1b+K7V-rJ;@*6kG^vX&X!C}2BDA!0+f)|nDi2L%XlDVB& zM{aBDYa47P`!IFYu>*%x0yu_~ONYInN`S$6QTxgT^rMvgS}f!$_LEc zqyy7Q+THts&101wPI9l*vfeTtqgW6%OAKsEMpPw7FhfCBQ;x6# z(AT&k6dl+c?C&<_HgZFg!d51Ab|31rqe`-DA}uqK5veIHGA3}kCw~}O*dOlnFjMMb zS(L#iMx(y5kdR%JHmN$P^U#l=nI}4iqsSw9X`Jp~P7iDn>>gNWuu+oPx10NWw9~$_ z<#~X;c$-(OX`akikF#Tvq)v|H&<<@@e=K37H$$QGSf`tclB_nXWKmW0b2;}*lN-&9RE%gk z9m4Jul_2S&^SjD%D({g;Rqvpak&}pK&X`{}(_b(>_UoyDUhk#egqX;rA0hg1Y`0|4 zvJIAQqTQZNb#5j9fYvY142C>@`MAk-Mut)ctt-i;ajknUdpC>Tk?lWZ26jbV+p!_?CHNO;VOHp z(r?g<(G5+tRUvOip{0pzo{3YS7A_SWD3C>l=bpZichaWF9EJ`#NU3>2hK~>OwafF{jMT%v^u+l>%I{$2 zu`Hr0s4gGI;={a*G+$n~=azjH`$v`J%Je2_!6>CCK1wIr<@Hs?u$`Vlc_s0q0+bRA^#p8yjS{DzjZvrNS z4~kXoNRcfI5R$Hd%7_|D<4^KeWNt*QL&8dDGJ_wu?P5%_-|2LRBUge_wux3z9ZR3K z*#?mhoeY5l21N8RMILJA)iu?o^^|6g>7<}>*H1~t;jtCf z>Z~euQ1L}yKFz^MBYun| zC*)xC==?9!lTg3TTO8Qfo8eT2pJhUEI`NxjkQI0`poL_P`wX)44)jIvk%6*EVXL5`dEx0mMWbFHFP}y&=G>AW>du=mw zdL$&WT*{F(RfkfZE31Dfk6(T@{-y}ii#1zVk>&4{9YaIT@gDOkW+not0`XNky2}c< zb86L$Y5x2fGAuStVZ{L|%WaDR3;hH?W_0VkH`EECpeCv}J0Y!hYOLs1kgg}3ZL4Is z?fk07IR&YChs@d~T>xZhwJft18wp+{f+*`_ znXHf2ajqr84EGKLO};~Ntv@Z3I7}Sm*$>zda8L=@jQu&YGpVh4k;&Xk@w&9o8tc*c z)<3t)Q>*$aPnIUh@EId=#W$IbqBo-2=bLr#(ZGj=!A(LA9KOY0@@dAK-pDer>tspj zWt9rLjJ1al0+bYlK4Bj<#b5J#^Ny4~P?UMPEAt@AS@r*QciusgT=#t^X--;6rX^Xj zWS6Q~E?B`ZmeaXgRW26*K1wJ-%Xx@Gn6sIS};avwX77Ujp-;gr4{hynd@GMY5uD3Ys<1=#OA||7=Ruc{B2yqhjs~t1;%ZbJz^`qB+G{k)sl~e{Kv=x< z$@poH3^h&P3*lskEauk-dm+%53yJiOHpiUK(l_(o$-UETOC&KcoNztGIz za=sLEhUGz=k@c55c>Zu-UenNtr8Qo#7Ba6rX*qaACJ0RNlrHruwhNc_!J1qkv=RxN zMyNT3FiHqo?%9x?w$9ivaUj&mY`unxRW zSKYUvbLHy{gB)hXFcj?;`WBLdawFPx3W`&bXp|P5 z?VZf_;^LSCHRa0Y3C57hgd|n>Z7#U=WJ@&sM(946TmSi2rP{F>Olu;r6AN7iw6i-4%IYCZqAqoH#CB7jlM+Eml75U zLn1Y030ZOsJeIb4NMngK+qA;pj19{<5;{~85tc-)m#Ur{8YZ?knF57O$aFLFCLQpL z5Po?I%2@>-;nf=(8o*V1u!Sw*ZqG>$+FlxxsO&=Xk9uKWJ+z^rWviN6Vk?lZ!-0-1 zz|$y2VJkeWp+QP3np>)1NN}9o9}-NYm!RVf)x+mD{Y6h|@26mh+3HZAnc&otbp|H; zh=zvAuIS@rjzee6UMUom0@X#t{*jF(5{OeTvM!O$r4a=qkefsf5&2j>YFQJdWQ=9c zswfhoAX5fz8PEaj8S2rin~?XItixS)igwC<<#a z?nAMI0m<3_#JPQJ_BAb3QIr~4+FO11$|)hN_&W9H61f_>$vWGwO!0cmbIp@~xh_x}k+CiWHrK8*>o3LWj>L zX?ydsO}d&ua#k#`SCO28$jS$`gS$y-uzK3EO}cDpsfwb78XG!{L`KO_Qm?6=zPhO; zDvA)~cfm*lm5yCeg;md3-4wRyA{9mKQF;upJi)ULh-E#qA%#G7dtqi;;tq@9AgY>gLg`hdrrRv#hwN*{0x-Cxt+D1&xMn#b@ z8@RUTtZr+WA|r1OMVOcXH76Ox1?stt*|5&+csaG*RtHUr8d_#Nid|^bM8FqvEk3NC zx9s4qrq_OojGO`xO`@PdeI(2RxqAMbw>#Uppz54{tFs;pd_ZEQ_ld`FD1h91!Fnyu zzlUC-@km)OwNi+55XzxK&Q~v7W(-u1yi$?TgsGsh7_Am9x*U z4H3Qaitxq)cP9{HT+rYYKtQa*etgTE|2Nz0++1Ji<&c;m{b>O*BAHK6=4Fi|Qe-5t z#W4$?Ojk6x3C_H{aa4+o02|?a4?#_bejlA_^@>9`T}+3GA|t%*Bxu2{X#KvvdS&Ct z6d6VR66I#39w}8I0Iy!vkdnK7_IF2{xcf~FWU{u$5;rFwK;0>&e1b4Rd``W(@xN9y zegH*Au>p`(M7|n3-9roJH4V{0+NbNKA|v_}fMr6+2r{_^ZWVoZFS&-S~IaH|Ni;_cVUL`YjQX_twU7I)wec#=ow$AFPd5`Kw~z7=5Vmr`27i4(tz4T{Inc`$*&8&Re+)zzg;N<}g=7l{r=>o%@3rDTR@Y+Vh3t0$iBuPfJ4b!GVSopwL#N`=$=ZEqy>U1* zeO#f0gA~jS5ZKS`d-aSXjsij$q*R`A^1vA!sLwWx;^D%3I~2Q7HN0DGrXh}x4Z#6< zO&Fah<^SBim&=gJ<^9upnZ!P=G|%yze2g>~)OJ^Weyz*PtvpP~D~9kDfD1vzAyi-3 z_i}mksk`ILlf$5loXbbb$urq@ucHyY~%DxTE2Q5g|VS^Iq#Qvi6F;-t)*?_FH9$}oV zfs|cyQfHBs!oq6#1Uzl^wYk`DZ44F#Jg6!a6cj!M99mHf!erkTWCkVUp|Am%yahgLr)zF1ni+3M%EGMxeV+}JDZa>ag63j}X8Xjz9 z-`Tf~g^>{(GHW<7r4Suc!t>R4*K4CXf#?Ss>oN4-1j9f8q*1EAw{IJq2fS9@Kyhr) zMkdn%w^al&gTV9D_xEjMB*i%T7%op2+9178u!xkHC3B2gydSJ?qv(b?h-ky79|1u| zA_zhmm8&1lB`0dH7CE!p4q0tw2$g6U60ow=kM?b(+$Z%`Q#C>%Jw>E&WuZ+A z#LQPe-nWsdQLP>a@epWF5+VXVwDpu=|72AoSOElM@(*%TpC9Zxp7k8Zu?9vDo$TT>_{61-)j6R`7zHci_f>Ti}`g`COkgK&R zn^(Wsw-ttVQCv#&<3q6CF?mf8$(j1)s#a=aJ9$HpGm%OIOhVS#1{U|Lx!7%OT$Tp5 zfGm;zCO(F03>m;!{d(VKmc@=l@hyW-g_fu&NkBKug(Yjf*)PyT`8e1~-Z^Kz4PAn| zswvbqb=PRpQt1^@{05^@3>p}YMKTxa>ZaieT4bN-xDeYis}^dwu1y@2NV*i6ChD5~ zt}7-e*HM&8ArLI1QY@bWxxPnJ5D68y24RC)-FU3QK?06ARUt;8)n-K2f$PJJ8VlptfZa`4xb1{jl@;=KWs|@L;|p8vK^-Z^_Z4= zz+qb=IOD=uk&E=CkZY+2HU+j|Rt@t~6eheG4UM@;D-#;GU8C8?qpiDZ0jC1EOLh&W zO;!o3lO4iU*Ed&@W^uxhc)&ISc}GYsRu5`QTxw~Qg`9$92qerhnQ~WXMDC^t+DuKm zoU;EeF*T{PCo7(*2R9`jKG`PhRolcdysC$J>=pju37#*>^^bJkH z0(oX{nQX~W=IN!kxp}evyN-e8H~M4gr^7#3aUP~Kp-O_3L~V&wsvDbvtvPhFUcNon zuwsk)au19FH7O{yb9gAGVq}FO z(w_*^EEtNL^xigoMAz+!3OZk3NOQZLEGTwHCTECflMQ(TMQZ3G0W^6`vxuU`{VjyE z4NXwDXZNhfHYdT)lNsDrr;Xwk*xAQ5XHi%a`^+2fZ4m^Mf|#OC$Jkv|dg3kN`o(ad z@TYowa~82Q*N9gaV>QiCVGQR6s5R`HZVtrc3C&rg{A@D_P1iJC#W^pdOfDr{lwn#v zDW@3yMD;|yvo1C~&4bys2V+~)u$3Ow0#d60!F5LBlc<&Ku<#Z?>F7ifF*?1+_&LIuMC1 zRu-xzeS9R+r8vXFj+3gVFK zw6QBjC=cvFwFxkn6bmKo;KZNV)QVt+!9kxGcOLAT6QjP)^>6sNc3{HB!j=MKVN+lsDycAvnP54b^j-caYH@54Vfb)Vu5oEPv$b z(sf3?8zp(#hb=jCyqGw#dR|lL7UT-&GX5XR0-zLn2=R#kTd4_xC#I(->R>k}AyTe; z2l)r2+&qW^bVYaJ_xPf7XNQ0oAV=~e(5K<^o9EPS;THQwBo;9m8Ng+kdHD4&X#R)V z*F0`D6e2*Pqz?WNpeuMmV`336)HZb4W6RkS28z z^^&G!fXmh$izo@c5HERKCzQAarNe2x=Y;-=n-XXQ;DXV!PtRXE-3?h-<#r5-ts~ik zH1w$ZkJK&Azo~VDa^AgOIkXPG1WvsO#u2LteD$*Cb*8x3m~Bi7%JX!EbEyeY{M$6K zx%uDZC0!B^_by`4>XX%+#v%wdDVEXlU<|?LrhHrY!DJ1Wsv<=2C_G~LLYwMhV#Y6T z3SX;?U%_C+26rMO+|DO>zR(=(Lbob!9MW$G3MFt$ko1Ug|BB}I`Z)^5-F=>;oHr6x zFf-_Ao(Ae7mI2Gs^B@hP0P2<1Tq~Z&qJBJ&$J@NQ!BCze@L(mwJ^{Zef{%fEmEL%2 z1K@T?dyMtSlpmggYhiE-3MV!X=xj4#c3~SXmjNb^*U#W_#d@Y>&sMM2k%U@vg)9sQ zzr5mJzNIT8`vm`_D2PgK5^24)mID{fiL zS;})0=*40fuyb-Rxq6+pRp+~Evw28Y^#tz4gt*Ht$Si=WBqVQ0U=Ab6cGc^3yk))> zI0bggTrn6;-%Y>Mc#7Rw?+`&$;WJ&N(?Y%+ePfZas&4*vw+`@Q9QpGWXbpVvbYFA=F!! zyx4LM>Iq@*;;v%lz#_OK{xCRVgpYxNaB_jKhLEUvjV$k?;NEOCY{IfPX?i1S?y<2- za-;djnEviv3B56?s;^6s%E&8H)(*jKL5oB#jNF_z=f(MWtrnhj|c&qo!bB7F<{m%#7IPbSjU190DB9QeL+3o~d^>B?G_U%3@s%vu!QM z%AgT22h|QX9u7P!WOlu)nR9LNo~L%iWBPtt(0k4)C&D-_K69z;*?Qj=n?Rz`Cb>lLj81DKzn7a2R=8Z{%maWz3ForS|PH0(n5{^N}4r(RgFndo^ zXt)&!leZgxA`96_IC{JM10dS6q( z+Vj1zVb_w)iBzQM&GtFznLB(?>~dsN(PJeuoWsosg{gY~vR%To!#FVrcvP7~>>x!k zLE?z&183bUs#@5$5Z~!k4d;YSn%fQy%+?kOy_M zvDH@}uZ}@;NpYHRiEFs=xDSOr0nvA4vtX4a!K6N+A6q85zR0!yo#kev6l^cb2 zNd%slqb==Irx2*mG;8dJdWVy@xw%;m+c7;nbK9%QD8aU{9iNtOlmV+0EDUJ_7wIUD zn@D|jWiUR+>LOx}`yRlLL?P4Xj&}Kez9}VpCIrL4&Z=1#6M$^c5M_FOaB4_X7~u~_ z*hXL=Y>JT87Y^4BbdOk8sf$MmFpDVD7ti|2|I%4r`Crxn9utriQdf)}|Cs*Ea)uc* zB7qSk-Z?Iy9>rJ~>o3IH-)sNiv~;3ocR^9JKx8Mdk7?$ZHU*$is4z5X3JOie&R7H4kqJbBjhtt`QG)v9B5vOh_pJC(()&C)-Xz4h1lP0zULP(HLc>GHl-051a7| z0%{XQx{=~pz=*c`M)U5~3an$8y)9JTM!vDPNlzH{h22SWx!5m8jK<-nXO8Yh^|Fp1a>LlAm!{`;Qy zUA;w5r)2CdBthUNG<8w+wk%J)D!qy%PN-F899qOU^q^|Bkfcb|_jF`&o#s}PQBkSQ z#*IR(n}NSe`s(`{rD1tn$(Z2F_O{{sAmt&E7#)aV>%zx_ab*kHdLd}NM8Y$d@=tjT z?KkN1V#g*G(Hn`{PDK|9@{|O!hj0W{(14)S56X@mbA3I>Pruh#I&FB0DK~)xgiykDE``o3c|dAj|OT zE%})I#DuEm(){u=8>Ga1_ePa?c>Q27*VguP{q}V8=MRU*bkRHI$#Scn93Mhe2VK4k zek9=F24Sa@)OO%AZa82%zx@v zzpFh3FiSvQWD8L0g_S`4`arA2B^XG-6{=rUQj!T=x>|m72JanD4va?#4kThkAcRMscRn9H21#$yfib_B4&i9-%13TMvOr#p;^Y6|Cz515xgk;%}En@A-nW z1WGf%oiBtb+EDS!X(0lDF1)l$XU0ABjl)A6Bic&S1kjuX!UO#eM3S{(C*A=6C*91a zJp#-aZa`9j>H$R;CP(}Wb*cAUhzya@2j+x=i>&M3Dg)E#)4>l&@=P4lY;BvmGI1-3H!A??YQ7VO!!t?t`03P9_%2?x1^?W>06hR|1n zzJYat8sQYh3lbd!GwyfB?ZVh16Nrd9MkA5Y$U=!r-T%zTfT0BN*`)*w5Svf3`2k1M zS0N4&(LXA9**YQWcwkGC!`7Pb%VuZT&W?-euaz1A*-p|UBylL!iP`6fk%Jsm*S92( zq^B$Joe??u%j8dp+6&Q_Ez$BjX|hoAYT!Xd^Ua0QlLO~LEkTseUu~J>>vr7NK;+=K!+6tL8dhi!>`u9Fa-aZ<8@tReN# z)}a-7fvh{yFJy1gbONRIu+~`y{64&OzPvyQ9C#yf%H0u(=|Z61+ugciuxD8(vve!o zaiHc9q0}z4P1oKS8OVKv*ne*Ru*o^1w}cxgn_XW$x+Q6y`L1iU zR-{j0={Z>{{tP7#ORkAA08oe`P(psTd4%ar2?7OW z>PaoZR?TkmRv_8wIAK>je9al-^d(LB3a^g~08|_;82-4TsJs zl%#T`o~-xGxm$$h)aSThcA+H*hOPdVhACwhg#E7E+VwF3n8 z)Kk|VB`GoXKmrpQ3tZPt)Xj%@#dsiSKw?pNa72Y+Up=iQMMP&SJjI*oJEX5d^vB?| z_0-c3zE?(+3B{<;5>2Ht*oW}xfLk5%b3$Z~)f4h3)ydCnNg>YsGXs-~hZeLhTbV0= zSfNZf%a>5xMMYfOvQ70N5fLCr%nkSg=@cjfs{Y{L1Vh=<46JhjzjazKFwfvG+FAQ8uhiEs=lmEdkm_V~WCh63mLEJWL*%_@p+N zgt)!HaX{ovaW>Kph%poUi`8?r!Gp7m*(}&HW7nsQu#vRejkhG39mk7S%%pO zCtd_Hl1w?;=ZJ#>ELJaTUCA6Sa)+y4X)O?b@Z2G_v~UReXj)_EfQ3*mpBVPWUi7rF z>X&6uJwc-^sX7#$3u+J{I-0qr=4~A693R`sf?7(GVH1?np#X7*0Qm0qm%HW6N6HZ@NhdRa&2L^X&IXl~8lY?%3QYaPU@QkO zkOyK%B6ujlJxT!Ivfw2b_b%a3ltmABsCB_&hA~;{aJynCkJWqP^69>$`w*{I|-usu6pN!`8P8-mZ=~Bhpu55 zaS&1z4hMFZgm*;T^Ia_gY3}!6`nmKPU|=xBEFt`XKQEVJ67}u{Z+E&2f%L#^03d^b zNu>g(415Y6)O)l?v)|nfjS5*06WQh58Mz>L_>tM*~r+J z=n&CFOTBN+A>~C&r=$s7;8>Wz39%$DWgzd@rug(jnF;Sy55ssEkDN6tgHzP(@N~si z2{nKX!BK+$f`C$_KF|^-hlP!5dh%kIts3nU#MAg2p3^i5x2s4%u%d~e!PliexaOk{ zh9JB3>de69Qk4u)lv?`?=9CZVDB^Ulrr*BkI2L<@oIGBLo^qGE8!GH!b_~>q*KFbB z*3NiO2XIIY+9-|@!l5Jy#et8k*+O-2S^+HL8i;`&22}*=qb;em-S;Ud2HfjSP8FU6 zs?JEXRzbN5+TZMsnk)#{DXGmsl(;zE>e5mykxv(M^y(f%)d zs*#oC9juKMtAKh&);|;xi;&8-){QQyM!V!KIk;Dmc->7s|UpUBU z7rba5G(sq$kc5G91j1a?70lAWivPkp)dSpA5F3s*iw+3J}*;++3C+5^NX zV-2S7G?31ddgx|cIM9(4C<7W`(o^`GtFIj9p@mdzgzKZt@Jtx5;yeqweKFKh_0_{P zDdSi*nNu01=X8u#fKU{DO$XnLg}MK$u`Sa~FC7zg(ZT&K!z(KkN+O6!1c@_(jp*w- z7`9%k_JA^+e0xCA-|?P2f3+2%%_5<4;ME=jn{?GTTDqXicc|^FK`ZXjNztgHA{L5jT+YaVJ14{-HR%1eLDe2RY z4K$C`ck9hBTuTR`O-Uj9CXY=uz5wT;k^J72k7v5=u!%YBVa|@J(V%t!)gmar5}%m5 zrM|BN69ojNd}0$mD9VBN?b2LT0|*I7T_Z5fR5mbw)DI4uXhkK89Qa`Hudu9({-R)&jwURc{)^P@37?tlC+&8fSW#+ZanJawhm2eU4uBla_FcOzkH#JtV5jVbl{_ z?-X0et_1#hMymG^5CPv$Wlp4i+Pd?KW^B{Uw5LI}f@v^DdkDk_@d)rN0l18xnE*R> z`;ek-IU$Q6lykut2X0gp&=UuasZ``IQFvfC_<8G@`t&Lwt#S-cnqV(u%>D@-R^p3W z;9WG@c-hcR=Hlk2UXd3M2AAJyFkIu>e;N`%m&4mCO0&u^3Zm{Q^k7|K&EQYSOBFgQU_yYG0+tBWFI%tIo%KwRdv;uPNx|l(40#oRoc+!nC;T#c*vkO- z*s*1}TB8pXe%h~7E}?> z&KafWiII;ks_0^LKA{q)Gy?DycsX^Cwt%jD6Ih>Z;4VAaUGmlYN6|Jy;;4n8Cgic{ zcPVtaXIps57PMU+nCx{+TCaAPsU@4ilLf31_mzqWB!|?sZ2;H5=h&q&a1w zFaS%=x5*Y*Jc6)riXmrv5fOVkQuk`7+#oQuIZe(|gx% zQ;O@(-mZF`>po3lQ9(kGmY6mJ2dv;e?Yqg-?7P2RFc@r?n@#OTvz;lwGP@r!ueunV z_@(`T;C802TjQRWLAg{mc!+zVlLJQBg|3BcEfZYbw>{!H!^M-j*vD}{*+I|XuGnkH zB@(`m2%;lXukOc+zU!T@xNv0skKm5+-(6@Wr;hi6UavPCqX@#h09O&RzNch14+Pz-|pTH#6atrTnF&m(_mfPB}g(9(?M9tO)jkZ9ixRq0HZl3Gw5k8J^rvYW>+~{J#fy0EPZ%8+^4V7 z0X;R%2>0|W7GPAmoedwsc$;1u3`3}&YoOUJ$a-Pp;ju$+2SD0ak2H^H2$C+H-)!TZ z)0>;;@91joVDI!UKrTrZhVasmWr0MIUQRSlhnziRH8Ju<#E!bTs~&y$W7+`; zhA{#=BIT?$5YwC5lE5rqpR7GSPA-xgTSQiL9={km%0K0?sB|P-Omr69o^WKT$Fx_m zv@Ts{GzZ58g@^F0P~PQAMbTs1M~AVoAfo{F7DKTF(GWTR$F&cOTOl_iBnvQuS-y>~ zt$IAOZ)V4`LSulllAxGOFnM*j3e-`Yf%%85&nDOIsV6Mm?6{IKfv+6)CI=#^P^2Yn z^~9wYnB7tM1Cd)waF89sSY7p`6&GNnN%)Mg4#f^6z|@ncF3@wp)4O3IP)xQ4szM_3 z@T^&|^Qxz`|4;4l^#$QU$cSg>1m*8TwiLMMsLzw_dRK=}OWTVoYfH#{J<^{!3GAn~ zHE~k0?aa(uTX#qNAmc>9c!|ITQVRWyyb+a?H@B}6SExL0^pP33)0mg4gT_{qbGvkR z^iDa)cV3n(Dkh@rJ?YBL9ivX>Q8Im@k&5dP^bg=fCxAB$g?PMsKJhK_7h?0{_T0uy>M-cvf4A`_I}F7v_`MQHW^*Da#R=Amd52Mm@VNsC{$zxuerN ziRGU^)CI$rmNof?adkuhT6zz+Cy;y2>^O(>FjCKH>zWZUvi4&K$heRg6TzlFX#lLE z=e7lKBTwxc(s+9-4KLMv@)xDD+2H=4h_MUx*4c$AjaY2%Xe%M^tLIJma|_M4nuTpl zz$NimbS?m13iw=91ke_Eep~bM$exnzT)~-JsO+Q$3>vd`fGs(s>A~dM>IE3L=1Ju1 z0g6z=-arC@V6c#EpiGNmWgwoGdZBh~<(b7s0{^rP+-zjI24SBR8r(dbl6aKrMwfM?UeuPzU)d%AoNaw(yAt{gCFm)f8&ockz*=51wH5pDmr#RYMM76A9K`0| zy~{HlBn8w+Av0+pPO6aIT)k9J#PT^N_M_!oP@0k#50cA0_)<0TBib26e_@TZsAxbd zTHVtAe+)!aIDIlB7-PVe<<&rE%O9m77~`ZR@?ge0u6{Xh*Daej1UUPYKOq?AP>j}B zFFTwG?4SnRO+euwcTMaa)yuZPGh`@gD+)1PF}Agnv@-Wgc3I$Y9{El;kKmquMOzTv zGfncA@wiG;cx9v1uEDZN$iU8H>b@NH%ELXcyfZ9nAPP2Q8OYtgN*iO+xfvsTb(b)A zgdG-0DE1V!kgu-HmuWA=#ibE^4Rk6o%2Ga_JC$10c)X@9l`3*?z5DslS&Rld9(AhO zm6l5tYj&mhikiZrY$WW$u0QunVWN81h{<515FBrIf^%ldQ&@QunW2x?oQHJZ}=jTKr=W`x&du(E?b5(nUFw` z+#29VE)BC(y}2!w5i*!X7lUH^wgS!KS#IV#@yTrU{K}x3pkBpsg?^pb`CAU}a#5z) z2R21z5v;IQH&JhGOKGw6bjdPj2ve*xqCPIiQ$MnkzIt2r>S`C6w`h)x6Xu;0=|@%t z{%N1M4Jl{!_G+=#E(CicJ3ulR%btpzC=RtI1uvNk(Qtxb2_nfpRMw<))H|w~x1^nr zoHs!PjYlQ{2?|IAsfbyDzfkXNYj)J>XC@bF=5En<0%|d*OBcFL8K_LXOWPPTBL&{` z^!5PtnJSA!f*r7D4y`y;>xAdjyQ}lI*4L@Kt(*a2$k~S26TUl=XJozKGaH;{tI%_G zK|o;1!4W&)j#<%E!KScVTI#*kt~Kdr0fW^G7?cs~7XlB+Ho=W|)%)6qEO;eAO6e}( zCC0}_6idDTur0YvFUmfl6akSBG+%w-uq_2bcn!fNGaQD%psPN3*p^_NOYrRIP1gYq z5vva!wk7J}EnFjsv(h3Isvm9(1i*wDd97M=X9lHTH!}4uRd*G)OlGWbV<}rhpOLcB zKD=(?r+-8T_@|pch!?$5(&wXkv#D-iBC9iLP0mEeW225Z-)s`S>SJw@Ln#=%^nNWd z(sH#LpHk29^V4`2!>~Fjmu%kuROVBsvK^`22L#WG6^uDhY4!0n`brLvDnO=R2pXE& zclC)iE@CgGNV7{R2?!4qOVuY=CZ|9c!4Q;Dnnke!2ypeOm6t%&$B74tBatRe&;04t zmw>(?n?yF>hcXGKqtC3oL@Fm@7y*5F6$JOGKHGkjIPg^@oTUC-D16O#SbdGdk~-bZ z%?tJ4I%r;FiJmP>kzjcdFh|)Xj*+7U-qM$6pN<^Wk9FOJ{Vg@LP6l&nSU-bR&+&?^6RV5uZ);3eJ+~AB0;8@ z0x{GMsKkX|Q~M3o1UQ61A8G)=MIi|9qbGvfs=nCXs=qa2Mo{nJ1@<&YrHs6uE82_@ zT0r69uK5(u4(gI4^FA&y2X-+f3Zn4~ocgFgC>~Ou#P`3jmi9 zNlB<(&=OT&Z;O6k0nb=G1D3=#W{qPg`k>Y)pw1ff$y9$sXQ~g;a?8uhvK zIu#r__7!UqP@TZvzoVTJ-<;?Rd;MQYsw_D*?>N(Q0#D-Xf==v0`vMf^yL!xh6Sp?| z6&qod%$(?Dt~YBEQ5GOL3QQ%y$?dD})!Uz}cL4%2pZ^|*u&noUqeI~-DO75@ z1YQ@qsrtU&WQzXl_I2cjpTyi4?`&NuyD8jBQ-oLgi}HoZ+?jnZpD~sfO>XA05>l zv^mHANhaQJ8&yRmAR$yca0V}l%c&_Xunfl3Z<+dR4F(C;HK_yEl$yYzs zIF7LP#`NZG4!2qiPG-OMKjjdd(CRMOc;*%Wy6DvKK^IKi5CsxX{mkGvOk!EjlG;7> zk7bZf76Q=ngafs_{gOpJcUP&FgbpbY#Fi`Dr4>-j7%3>!fM+9rjK{uSaj7T_n9++T&%|t0#u3xtQFUvnV zo3{?NloUoR_b1VT0wmRZ$iI{HiPf*>9$g}SywxgTIKPc*ZoL@S=>!3?)Ub7vZ7C}* z?pPEXl=df~P)X1C#m^`G5CQ6jf*+`%PBTRXx!=sFA9#}OKkmYiO@S$;L2g$HJk@W^ zxS!pdPfRNG4*(N@k$_zfyTnmfb#!vz?B2ir=21fxiW`Dv88m%qs;=(nnYY#&Uu(7P z{{|=}q%J)s@G6jg-oP~-Eu0gb^$yfWfrNWRX+Oz!G%0;*e(upZWZ+83?=48jBS;3Y z1o<;?L|1T!}ohKqD%+slO(MfrSnrQ{B6xsj8M<7?8aip{LEY z_>nPcTf)|W)|Z>9`{>c|&HR=Y2&vaQtv%YEr8e4(g_%R-&5-kx=W>r8z2vp^Z`Z?MTvhpLQZ` zcuNL>%QX{)C~`zj2Khj$?$>DwqxB%Uf2W;pn;#$0`2$pggc8vtN*~y{Z9dGh>pRVK zTYh*@rzyntgXF=TR)KAMNaqhyjz`!pddZ3L&gD0B{-w}l!cLkAdFJt*8#{kkCOWIn z9@=SY3+al7b#B+|p_Yys9Wy|-?}v99`d5$WGEAnGZ$5dnCVlOy^IEUzsjRf8_JWN-w1k zy*xGOkH>ZXRq=^%LH05_`uI*mr*)suX>v(&Dt#h@M{)F%I=3D5hnGKDe@w@p!nkKA zlasqo?fhO(veZ-r?Toi??qGY4_I8uq(@*O(<>TGcPuD-BgJiY!*a?AW_g$A=%hw;fI0a4KP z;mphP0Cy!-&+ptMktTA=b%Bh3hC!tz_Sei>wjllEAYwTS-6sjM7jzzL#>boldTj_N z(LcMO+mi8_GmPDU(}@~vylGJ)2-OQat4bD#AcJWa@l48b+5XjwI;*gc#W*v)b(ri} z6`^Df`wlFq35=+~kz(j0)QdZku0LXh`j@Z}4;5CLIM~x=P;PTkOTpn;d z3h>jii!DoUd0a(G0M1(qwSn-4SQ6?L9YODytDHRF@-zr2xX?@wD>AejG&QAJ5W8W9 zKa)clj=~=-)vxRv7S0OBIR(~%NHK7xEP}mBn=fULPVN=;dX^375%MTvzYqxXDBPGB z6p&=JFSE=BdeIRQ)#5^?`LcRo8yW+!WDup995LB@8!$h0C}=ApJ)&Nt*HY1tjH3UK zM#AFK3XvuXGJ8@7JugE;2~*}t9g!d8B)${%+S(9njE7^#!$dzu59zJJcn`_SA*iiw z1dh!IJYHb?ga<*#rU-k$d#qm95ty&(RrP)sT0LPZ#Y||nq3Me}f}>vF5lWDmOCzK8 zBGH5Dxk3@XKv7jpJe!cMlak9u(G;{wSG}Plz=opAXz^|`x8q?G?BFO~D1V8y>*)_@ z9lJB22tPs;!2_3x71#E>dl?L+``CWnJ8#+BlA)>H!s$28zX$W1krqV zVUQSh2CQr#F!0I-l&q__bWU@VA|ooLX;V3{+V~f@W=WID5|fFyU>5b}Cg4Uqeqecb z&Q4tas-(N#rLV6*e~c3e_?5Lb{Reyd(!u8CbpCF=YuS@;lel~H3s zibe9I2qmevb^elvFnJ=b@XE*^g#ZK%EaY&>d4f__Z`UVA^+lcZHZI0YK2G2FGB`7b ze-BK)p#aEl5im@;ZZxX*h*Of^m*e}|$0wq5+6 zcXle`WzBVlue5h!kdExxB)SZn47X2CDW*aL2WosAV_2d^fEh6Y0*hD;(c;uq@9GFb z$2>o;5OOf##xB5BIWp;7M4Xh&E!a_qV*3>4-*=y>V>*QtpwQhxgpJ$~5Sj$;vG*L- zgC-Z$ht&tb0Xie3`P6&$j#jr+xKd)X?>lTK z5Q-%x28P9_@Q1ulDVn_r^ zrR2+3AM8kB+Jez?7%b)i5IcBMVg3qoG7apgAJS*CYt~DjYQwef?b?)9cyO{tQewf@ z_ThR{`>p)B$H0Q>!ctD2mC#8@?a@a%6VeV=tt@GW#XNgh7lqtbx5t;NGv8rXK=Ud? z@;ZT#53;NJwr(K*iW$N(BXc{ymJ{; zncVWCW;e4j;A6d+O}`~VH=&){q#hAP0fVkS-TB+{877ZiVmA9=|I!hG5CpMwKO|!& z;)7B@#5DfQniIyX!s(<*nCXbvEhIE4B`ZE2tes9SdGBOg>+IUfUD%_n19VbSR!T zR3XWy?lFz~Voafz2U3YFs!JGCec@&Gj`hhIJMT={z*qn;x72vPJl^dZx0B)2!eB+Go zP;P(49qfQY9tuaHY9&7oh4D8Jbccm+VS4rEOqHx}Q|;nGAI~=Qtuw#F^s0g%8l-U_ zN-sWCJG*=3qbqL4D^4?l85UkQfAcmXO`B@mz@7fA>IlSU6dy z-vgmK>KLGZh2A41IiS9G=65Izo?tjE#_Ym4ZiuNAm!^Db;SzJPdLkRg<#tDkqI$k>#v?z=4FOh(4;THzpH@B^H)RYkcVTm2(ojg(SNB+UxutK83o}mks=HGd7g&)f1opZW$rm|@WLI{CBRMz3+3c# zKUELd5PU%Ou4`=&)vtyQ`2?Bi{8dib96tqEHXAK|dg7`FE*lMC8MT3&YY6&JJIF)e z6o^6xRIXfqkcR9@kMPe34Z>NCd0~mz!Gkuml;`|xnp*h8=fKNS=|(Py;+aT2c%u@- zqn3To&_NH4!okpT0U{7!BQ~PLB)pXf^22 zt8Um3!ZY(q=6iq~Ha%eLhuVE*4P4+-d=KLx7&?gSkP5hQV-iG~Z@YTBqI2e6q&CL8 zG68M!dKr8_t_vloXg@x5#K)t+ZFI$7G%9d`fQ9uD z+Yng@_VGv5ju9q|-v=NwJ{exHb=b{f`2$ZqdPC?ZT11XDc6WWd}VLfhJF~;|L9)%o5FD>{eerWj$4%7EN_Ht$a%2F1NFEK(czlwNs)Lw=qe~ovHH4$ zH-ySc;jKg;06d?1e0{H*ycT~uJGnKK?jd7LA%0I=x^&qggMi`(ei@Sl0t)qn`d)Vs zr@;mHHQMTLpYD|}hJpvkdt;B2ErTP`Q%~HekR22M#i|OYcSd_Q830p7%|EnkO#iel zq^7H|euMzMv5Tnnhd5F_X=5$A6dQaA@mEGK06*+_>d6}yOudy^zo;;=qhdM1Jx4G* z-l$PZnq65VnE`?w$Kt-L{3Y4%i_$A4h`lU3BFDb` zUaTV-P~yofm`Dhq(ZELiv1Y#TOtXm$K9Z}aV(n~gz|qO;fcxP*L(;LfM{xnmjQHMg z$Fh#?4UZw>LvVap@Lys3kuisY1)o*8AJsD^*Sblzy=}9=WH!Xxa>SnQ8L>A#M2?Hm zXh9Dc0?N)|DSzfhjn>l)ku(8Q$$%vFDcC3WO;*omd&u-iYyjc~ED>Zxo;4X9-8rW_ z-8pL4k+PlYB;fHr$0B)q5}ObYsAq3z+<^&u&v)`*JqV>f+OQl<{sqtj9x#{F22pon^HHRpH|f#78otGQ#$!j*rgz2GU|VzkOOIk! z0cp3qj{^@nK|O!+8m5}ma=1o>*0M%-LmCa;$)TIIW|X4>!t zl6eAS()`VYt7H*kL`Gr3N8(G|xk$ZmLsY)>AWvOOmi(k(Iq9>9Vi5)zk&8Y=6D;2s zEm-r0XLhpYBWP=U;1Sr}z?uZNR=cQ>7oFQEU@r=UQ$(bjAQth}OD0EO`M#{_DPWnD z*@%s26qfKI*$h(3jL|g0;060hx#~+d?ovMJlBO_aIsF)^!B9_!naPyEml1+|rZ}9j zgWl?viD$F8MaoP^tnQh1JXH81YHe8M3@G64mmTmJ8D=MYlVb!`sl=sE5hq>*<+ytJ zB(P9Bl_K@|4_-|QU&TY8;QJ#`VCAy zP*v2kIQ(?SsZ-ISQ>9TT+m8jQme-Y>6wH3qSH5=RYVm81@-{1dMi4QeMTi6^MA9=t zj<4JJ$0O(io-);sHl$x5%iKN%W$N`C0&DdRn%SIBdU9yYrI{QDJEKG+7HBVo=&0ku zDWu-8Aw}><6EB>Z5yU0YJU|e`7V9a3VQC|ty%(&;o11!fm$k_%m5pA0T-RjsU@xZJ zR^l0~e@ne-Bj+B0L8h1y!yN#ZHrNsVALJ+oJO7g(3^wrqwiz3{7X}swGW!)gQ*YkT zXu0~U(T=%PUjWMM^~X@f^KeD}5Sn!UDR=GdV-Z%1YbpXGEKr=2grgu|#xPWG*&tCP81nPmHz;7>JN(%Cqg&sKiV0+DsOx|POnB}(e|Vc) zwKj(XF+*^6XL|TX>W_XIwd)`x$RwHD!kRdxX`k0_4`s@L8VTX zp4UHpn}!eyCx2mWM((C{`@aTY7DjN2qQuZ(m{lIFGyhZ>VK&&qjp@i1)|NMIzg$mIy6I zA7a8Rho6jFF^egl`I}ok`PR4Fi-#0?^ztm;qb6n4IrsPa Date: Thu, 14 May 2026 17:25:35 +0800 Subject: [PATCH 02/19] =?UTF-8?q?feat(plugin-wasm):=20=E5=AE=8C=E6=95=B4?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=20proxy-wasm=20v0.2.1=20spec=20=E5=B9=B6?= =?UTF-8?q?=E8=A1=A5=E9=BD=90=E7=AB=AF=E5=88=B0=E7=AB=AF=E9=AA=8C=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 补全所有缺失 host fn：shared_data/queue/metrics/properties/buffer/status/ grpc(Unimplemented)/foreign_function(NotFound)/log_level/header_map_size 等 - 新增 shared.rs：进程级 SharedData(CAS)、SharedQueue、MetricRegistry - 扩展 abi.rs 枚举到完整 spec 值集 - 重构 shell.rs：长生命 Vm + Arc> 复用，取代每请求新建 - 实现 proxy_on_tick：后台 50ms 颗粒度 tokio 任务驱动 - Vm::new 改为同步；process() 起始加跨请求清理 - 新增 3 个 proxy-wasm-rust-sdk guest 插件用于验证： spec_test_guest(20 scenario)、sdk_examples_guest(4 mode)、on_tick_guest - 新增 4 个集成测试文件： spec_compliance / sdk_examples / http_call(mock server) / on_tick - 全部 13 test passed，clippy + workspace check 干净 Co-authored-by: Cursor --- Cargo.toml | 6 + crates/plugin-wasm/Cargo.toml | 8 +- crates/plugin-wasm/src/abi.rs | 222 ++- crates/plugin-wasm/src/config.rs | 18 +- crates/plugin-wasm/src/host_fn.rs | 1496 +++++++++++------ crates/plugin-wasm/src/host_state.rs | 50 +- crates/plugin-wasm/src/lib.rs | 69 +- crates/plugin-wasm/src/shared.rs | 269 +++ crates/plugin-wasm/src/shell.rs | 100 +- crates/plugin-wasm/src/vm.rs | 436 +++-- crates/plugin-wasm/tests/http_call.rs | 179 ++ crates/plugin-wasm/tests/on_tick.rs | 104 ++ .../tests/on_tick_guest/.cargo/config.toml | 2 + .../tests/on_tick_guest/Cargo.toml | 21 + .../tests/on_tick_guest/src/lib.rs | 46 + crates/plugin-wasm/tests/sdk_examples.rs | 265 +++ .../sdk_examples_guest/.cargo/config.toml | 2 + .../tests/sdk_examples_guest/Cargo.toml | 23 + .../tests/sdk_examples_guest/src/lib.rs | 207 +++ crates/plugin-wasm/tests/spec_compliance.rs | 236 +++ .../tests/spec_test_guest/.cargo/config.toml | 2 + .../tests/spec_test_guest/Cargo.toml | 23 + .../tests/spec_test_guest/src/lib.rs | 338 ++++ docs/CODE_REVIEW.md | 222 +++ 24 files changed, 3619 insertions(+), 725 deletions(-) create mode 100644 crates/plugin-wasm/src/shared.rs create mode 100644 crates/plugin-wasm/tests/http_call.rs create mode 100644 crates/plugin-wasm/tests/on_tick.rs create mode 100644 crates/plugin-wasm/tests/on_tick_guest/.cargo/config.toml create mode 100644 crates/plugin-wasm/tests/on_tick_guest/Cargo.toml create mode 100644 crates/plugin-wasm/tests/on_tick_guest/src/lib.rs create mode 100644 crates/plugin-wasm/tests/sdk_examples.rs create mode 100644 crates/plugin-wasm/tests/sdk_examples_guest/.cargo/config.toml create mode 100644 crates/plugin-wasm/tests/sdk_examples_guest/Cargo.toml create mode 100644 crates/plugin-wasm/tests/sdk_examples_guest/src/lib.rs create mode 100644 crates/plugin-wasm/tests/spec_compliance.rs create mode 100644 crates/plugin-wasm/tests/spec_test_guest/.cargo/config.toml create mode 100644 crates/plugin-wasm/tests/spec_test_guest/Cargo.toml create mode 100644 crates/plugin-wasm/tests/spec_test_guest/src/lib.rs create mode 100644 docs/CODE_REVIEW.md diff --git a/Cargo.toml b/Cargo.toml index 378bf01c..8b6d228d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,12 @@ members = [ "examples/socks5-proxy", "examples/mitm-proxy", ] +exclude = [ + # proxy-wasm-rust-sdk guest 用作 host 集成测试参考插件，目标三元组 wasm32-wasip1，不属于主 workspace。 + "crates/plugin-wasm/tests/spec_test_guest", + "crates/plugin-wasm/tests/sdk_examples_guest", + "crates/plugin-wasm/tests/on_tick_guest", +] resolver = "2" [profile.release] codegen-units = 1 diff --git a/crates/plugin-wasm/Cargo.toml b/crates/plugin-wasm/Cargo.toml index 4c4159bd..a3a7d8c5 100644 --- a/crates/plugin-wasm/Cargo.toml +++ b/crates/plugin-wasm/Cargo.toml @@ -40,5 +40,11 @@ bytes = { workspace = true } # moka 同步缓存模块（Module 编译产物按 url 缓存） moka = { version = "0.12", features = ["sync"] } +# WASI random_get：用 OS 级 RNG（getrandom 是 rand 的底层，体积小） +getrandom = "0.2" + [dev-dependencies] -tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "time"] } +# 集成测试 mock HTTP server / inner.call mock service 需要 hyper 1 + hyper-util。 +hyper-util = { workspace = true, features = ["tokio"] } +http-body-util = { workspace = true } diff --git a/crates/plugin-wasm/src/abi.rs b/crates/plugin-wasm/src/abi.rs index 18d562e6..94a186bc 100644 --- a/crates/plugin-wasm/src/abi.rs +++ b/crates/plugin-wasm/src/abi.rs @@ -1,7 +1,8 @@ - //! proxy-wasm ABI 0.2.x 的基础类型与内存/编码工具。 +//! proxy-wasm ABI v0.2.1 的基础类型与内存/编码工具。 //! //! 主要分三块： -//! 1. `Status` / `Action` / `MapType` / `BufferType` / `StreamType` / `LogLevel` 枚举 +//! 1. `Status` / `Action` / `MapType` / `BufferType` / `StreamType` / `MetricType` / `PeerType` / +//! `LogLevel` 枚举（按 spec 1:1 完整覆盖） //! 2. `MemoryHelper`：通过 `wasmtime::Memory` 安全读写 guest 线性内存 //! 3. `pairs`：proxy-wasm 头部 (k, v) 列表的二进制布局编解码 //! @@ -11,18 +12,23 @@ use crate::error::WasmHostError; use wasmtime::{Caller, Memory, StoreContext, StoreContextMut}; // ───────────────────────────────────────────────────────── -// 枚举：proxy-wasm 0.2.x ABI（仅列出我们用到的子集） +// 枚举：proxy-wasm v0.2.1 spec §Types // ───────────────────────────────────────────────────────── -/// `proxy_status_t`：所有 host fn 的返回值。 +/// `proxy_status_t`：所有 host fn 的返回值（spec 完整 10 个值）。 #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(i32)] pub enum Status { Ok = 0, NotFound = 1, BadArgument = 2, + SerializationFailure = 3, + ParseFailure = 4, + InvalidMemoryAccess = 6, Empty = 7, + CasMismatch = 8, InternalFailure = 10, + Unimplemented = 12, } impl Status { @@ -49,93 +55,181 @@ impl Action { } } -/// `proxy_map_type_t`：头部映射的来源。 -/// -/// 与 proxy-wasm-cpp-host 一致： -/// 0 HttpRequestHeaders / 1 HttpRequestTrailers / -/// 2 HttpResponseHeaders / 3 HttpResponseTrailers / -/// 6 HttpCallResponseHeaders / 7 HttpCallResponseTrailers +/// `proxy_map_type_t`：头部映射的来源（spec §Types 完整 8 个值）。 #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MapType { - HttpRequestHeaders, - HttpRequestTrailers, - HttpResponseHeaders, - HttpResponseTrailers, - HttpCallResponseHeaders, - HttpCallResponseTrailers, - Unknown(i32), + HttpRequestHeaders = 0, + HttpRequestTrailers = 1, + HttpResponseHeaders = 2, + HttpResponseTrailers = 3, + GrpcCallInitialMetadata = 4, + GrpcCallTrailingMetadata = 5, + HttpCallResponseHeaders = 6, + HttpCallResponseTrailers = 7, } impl MapType { - pub fn from_i32(v: i32) -> Self { - match v { + pub fn from_i32(v: i32) -> Option { + Some(match v { 0 => MapType::HttpRequestHeaders, 1 => MapType::HttpRequestTrailers, 2 => MapType::HttpResponseHeaders, 3 => MapType::HttpResponseTrailers, + 4 => MapType::GrpcCallInitialMetadata, + 5 => MapType::GrpcCallTrailingMetadata, 6 => MapType::HttpCallResponseHeaders, 7 => MapType::HttpCallResponseTrailers, - other => MapType::Unknown(other), - } + _ => return None, + }) } } -/// `proxy_buffer_type_t`：缓冲区来源。 -/// -/// 0 HttpRequestBody / 1 HttpResponseBody / 4 HttpCallResponseBody / -/// 6 VmConfiguration / 7 PluginConfiguration / 8 CallData +/// `proxy_buffer_type_t`：缓冲区来源（spec §Types 完整 9 个值）。 #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BufferType { - HttpRequestBody, - HttpResponseBody, - HttpCallResponseBody, - VmConfiguration, - PluginConfiguration, - Unknown(i32), + HttpRequestBody = 0, + HttpResponseBody = 1, + DownstreamData = 2, + UpstreamData = 3, + HttpCallResponseBody = 4, + GrpcCallMessage = 5, + VmConfiguration = 6, + PluginConfiguration = 7, + ForeignFunctionArguments = 8, } impl BufferType { - pub fn from_i32(v: i32) -> Self { - match v { + pub fn from_i32(v: i32) -> Option { + Some(match v { 0 => BufferType::HttpRequestBody, 1 => BufferType::HttpResponseBody, + 2 => BufferType::DownstreamData, + 3 => BufferType::UpstreamData, 4 => BufferType::HttpCallResponseBody, + 5 => BufferType::GrpcCallMessage, 6 => BufferType::VmConfiguration, 7 => BufferType::PluginConfiguration, - other => BufferType::Unknown(other), - } + 8 => BufferType::ForeignFunctionArguments, + _ => return None, + }) } } /// `proxy_stream_type_t`：`proxy_continue_stream` / `proxy_close_stream` 参数。 #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum StreamType { - Request, - Response, - Unknown(i32), + HttpRequest = 0, + HttpResponse = 1, + Downstream = 2, + Upstream = 3, } impl StreamType { - pub fn from_i32(v: i32) -> Self { - match v { - 0 => StreamType::Request, - 1 => StreamType::Response, - other => StreamType::Unknown(other), - } + pub fn from_i32(v: i32) -> Option { + Some(match v { + 0 => StreamType::HttpRequest, + 1 => StreamType::HttpResponse, + 2 => StreamType::Downstream, + 3 => StreamType::Upstream, + _ => return None, + }) + } +} + +/// `proxy_metric_type_t`。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MetricType { + Counter = 0, + Gauge = 1, + Histogram = 2, +} + +impl MetricType { + pub fn from_i32(v: i32) -> Option { + Some(match v { + 0 => MetricType::Counter, + 1 => MetricType::Gauge, + 2 => MetricType::Histogram, + _ => return None, + }) + } +} + +/// `proxy_peer_type_t`（TCP 用，暂不调用但保留类型）。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[allow(dead_code)] +pub enum PeerType { + Unknown = 0, + Local = 1, + Remote = 2, +} + +/// `proxy_log_level_t`。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +pub enum LogLevel { + Trace = 0, + Debug = 1, + Info = 2, + Warn = 3, + Error = 4, + Critical = 5, +} + +impl LogLevel { + pub fn as_i32(self) -> i32 { + self as i32 } } /// `proxy_log` 的 level（tracing 转换用）。 -pub fn log_level_to_tracing(level: i32) -> tracing::Level { - match level { +pub fn log_level_to_tracing(level: i32) -> Option { + Some(match level { 0 => tracing::Level::TRACE, 1 => tracing::Level::DEBUG, 2 => tracing::Level::INFO, 3 => tracing::Level::WARN, - _ => tracing::Level::ERROR, + 4 | 5 => tracing::Level::ERROR, + _ => return None, + }) +} + +/// host tracing 最大级别 → proxy_log_level_t（用于 `proxy_get_log_level`）。 +pub fn host_max_log_level() -> LogLevel { + if tracing::enabled!(tracing::Level::TRACE) { + LogLevel::Trace + } else if tracing::enabled!(tracing::Level::DEBUG) { + LogLevel::Debug + } else if tracing::enabled!(tracing::Level::INFO) { + LogLevel::Info + } else if tracing::enabled!(tracing::Level::WARN) { + LogLevel::Warn + } else { + LogLevel::Error } } +// ───────────────────────────────────────────────────────── +// WASI 常量子集 +// ───────────────────────────────────────────────────────── + +/// `wasi_errno_t`（spec §Types 中的子集）。 +pub mod wasi_errno { + pub const SUCCESS: i32 = 0; + pub const BADF: i32 = 8; + pub const FAULT: i32 = 21; + #[allow(dead_code)] + pub const INVAL: i32 = 28; + #[allow(dead_code)] + pub const NOTSUP: i32 = 58; +} + +/// `wasi_fd_id_t`：stdout / stderr。 +pub mod wasi_fd { + pub const STDOUT: i32 = 1; + pub const STDERR: i32 = 2; +} + // ───────────────────────────────────────────────────────── // MemoryHelper：guest 内存读写（按 host fn 单次调用的生命周期使用） // ───────────────────────────────────────────────────────── @@ -191,7 +285,14 @@ impl MemoryHelper { Ok(()) } - /// 写入一个 little-endian i32 到 guest 内存。 + /// 读 little-endian u32。 + pub fn read_u32(&self, store: StoreContext<'_, T>, ptr: u32) -> Result { + let bytes = self.read_bytes(store, ptr, 4)?; + let arr: [u8; 4] = bytes.as_slice().try_into().map_err(|_| WasmHostError::MemoryOob { ptr, len: 4 })?; + Ok(u32::from_le_bytes(arr)) + } + + /// 写入一个 little-endian u32 到 guest 内存。 pub fn write_u32(&self, store: StoreContextMut<'_, T>, ptr: u32, value: u32) -> Result<(), WasmHostError> { self.write_bytes(store, ptr, &value.to_le_bytes()) } @@ -217,7 +318,6 @@ impl MemoryHelper { pub fn encode_pairs(pairs: &[(&[u8], &[u8])]) -> Vec { let count = pairs.len() as u32; - // 估算容量：头 4 + 每对 8 + 每对 (k+1+v+1) let mut cap: usize = 4 + pairs.len() * 8; for (k, v) in pairs { cap += k.len() + 1 + v.len() + 1; @@ -240,7 +340,14 @@ pub fn encode_pairs(pairs: &[(&[u8], &[u8])]) -> Vec { /// 解码 `proxy_set_header_map_pairs` 写入的字节流为 (key, value) 列表。 /// /// 严格按编码格式校验长度；不合法直接返回 `None`，由 host 端转 BadArgument。 +/// 空 map 允许两种编码：空 buf（`size=0`）或单 `0x00` 字节（spec §Serialization）。 pub fn decode_pairs(bytes: &[u8]) -> Option, Vec)>> { + if bytes.is_empty() { + return Some(Vec::new()); + } + if bytes == [0u8] { + return Some(Vec::new()); + } if bytes.len() < 4 { return None; } @@ -264,7 +371,7 @@ pub fn decode_pairs(bytes: &[u8]) -> Option, Vec)>> { return None; } let key = bytes[pos..pos + ks].to_vec(); - pos += ks + 1; // skip \0 + pos += ks + 1; let val = bytes[pos..pos + vs].to_vec(); pos += vs + 1; out.push((key, val)); @@ -278,3 +385,18 @@ fn u32_from_slice(bytes: &[u8], pos: usize) -> Option { let arr: [u8; 4] = s.try_into().ok()?; Some(u32::from_le_bytes(arr)) } + +/// 把 property path（`\0` 分割的多段字节流）拆成 segments。 +/// +/// spec §Serialization: "Host implementations should tolerate a NULL character at the end". +pub fn decode_property_path(bytes: &[u8]) -> Vec<&[u8]> { + let trimmed = if bytes.last().copied() == Some(0) { + &bytes[..bytes.len() - 1] + } else { + bytes + }; + if trimmed.is_empty() { + return Vec::new(); + } + trimmed.split(|b| *b == 0u8).collect() +} diff --git a/crates/plugin-wasm/src/config.rs b/crates/plugin-wasm/src/config.rs index af433de2..6c96b99f 100644 --- a/crates/plugin-wasm/src/config.rs +++ b/crates/plugin-wasm/src/config.rs @@ -24,7 +24,7 @@ pub struct WasmLimits { pub struct WasmPluginShellConfig { /// `file://`、`http(s)://` 或本地路径。 pub url: String, - /// 传给 guest `proxy_on_configure` 的配置：可为 JSON 对象；序列化为 YAML 字节给 hai 系插件。 + /// 传给 guest `proxy_on_configure` 的配置：可为 JSON 对象;序列化为 YAML 字节给 hai 系插件。 #[serde(default)] pub plugin_config: serde_json::Value, #[serde(default)] @@ -40,6 +40,19 @@ pub struct WasmPluginShellConfig { /// 创建时是否尝试用占位 linker 实例化一次（尽早发现链接错误）。当前实现已弃用，保留兼容字段。 #[serde(default = "default_validate")] pub validate_on_create: bool, + /// 暴露给 guest 的 `plugin_name` well-known property（spec §Properties §Proxy-Wasm properties）。 + #[serde(default)] + pub plugin_name: String, + /// 暴露给 guest 的 `plugin_root_id` well-known property。 + #[serde(default)] + pub plugin_root_id: String, + /// 暴露给 guest 的 `plugin_vm_id` well-known property；同时用于 `proxy_resolve_shared_queue`。 + #[serde(default = "default_vm_id")] + pub plugin_vm_id: String, +} + +fn default_vm_id() -> String { + "default".to_string() } fn default_validate() -> bool { @@ -55,6 +68,9 @@ impl Default for WasmPluginShellConfig { clusters: HashMap::new(), limits: WasmLimits::default(), validate_on_create: false, + plugin_name: String::new(), + plugin_root_id: String::new(), + plugin_vm_id: default_vm_id(), } } } diff --git a/crates/plugin-wasm/src/host_fn.rs b/crates/plugin-wasm/src/host_fn.rs index 3b0f764b..44d6e803 100644 --- a/crates/plugin-wasm/src/host_fn.rs +++ b/crates/plugin-wasm/src/host_fn.rs @@ -1,14 +1,12 @@ -//! 把 proxy-wasm 0.2.x 的全部 host fn 注册到 `wasmtime::Linker`。 +//! 把 proxy-wasm v0.2.1 全部 host fn 注册到 `wasmtime::Linker`。 //! //! 实现策略： //! //! - 全部使用 **同步** `func_wrap`（host 端不需要 await）。 //! - `proxy_http_call` 是唯一的"异步"——它**同步**返回 token，把真正的 HTTP 调用 `tokio::spawn` //! 出去，结果通过 `dispatch_tx` 投递回 Vm 状态机；Vm 主循环 await。 -//! - hai-process-mix 没用到的能力（grpc_*、shared_data、foreign_function、queue） -//! 全部 stub 为 `Status::Unimplemented`（i32 = 12）以避免 wasmtime instantiate 失败。 -//! -//! 命名与 proxy-wasm spec 完全一致；参数按 i32（线性内存偏移/长度均为 i32）。 +//! - gRPC / 外部函数：进程内不接 gRPC client / FFI 注册表，返回 `Unimplemented` / `NotFound`。 +//! - 命名与 proxy-wasm spec 完全一致；参数按 i32（线性内存偏移/长度均为 i32）。 use std::time::Duration; @@ -17,29 +15,57 @@ use http::{HeaderMap, HeaderName, HeaderValue}; use tracing::{debug, info, warn}; use wasmtime::{AsContext, AsContextMut, Caller, Linker}; -use crate::abi::{decode_pairs, encode_pairs, log_level_to_tracing, BufferType, MapType, MemoryHelper, Status, StreamType}; +use crate::abi::{ + decode_pairs, decode_property_path, encode_pairs, host_max_log_level, log_level_to_tracing, BufferType, LogLevel, MapType, MemoryHelper, MetricType, Status, StreamType, +}; use crate::host_state::{HostState, HttpCallResult, LocalResponse}; +use crate::shared::{ + metric_define, metric_get, metric_increment, metric_record, queue_dequeue, queue_enqueue, queue_register, queue_resolve, shared_data_get, shared_data_set, MetricOpResult, + QueueOpResult, SharedDataSetResult, +}; -/// 把所有 hai-process-mix 用到的 host fn 注册到 linker。 +/// 把所有 proxy-wasm v0.2.1 host fn 注册到 linker。 /// /// `dispatch_tx` 用于把异步 HTTP 调用结果发送给 Vm 状态机。 pub fn register_all( linker: &mut Linker, dispatch_tx: tokio::sync::mpsc::UnboundedSender<(u32, HttpCallResult)>, ) -> Result<(), wasmtime::Error> { - // ─────────── proxy_log ─────────── + register_log(linker)?; + register_clock_and_tick(linker)?; + register_context_control(linker)?; + register_stream_control(linker)?; + register_buffer(linker)?; + register_headers(linker)?; + register_status_and_local_response(linker)?; + register_http_call(linker, dispatch_tx)?; + register_shared_data_and_queue(linker)?; + register_metrics(linker)?; + register_property(linker)?; + register_grpc_unimplemented(linker)?; + register_foreign_function(linker)?; + Ok(()) +} + +// ───────────────────────────────────────────────────────── +// Logging（spec §Logging） +// ───────────────────────────────────────────────────────── + +fn register_log(linker: &mut Linker) -> Result<(), wasmtime::Error> { linker.func_wrap( "env", "proxy_log", |mut caller: Caller<'_, HostState>, level: i32, msg_ptr: i32, msg_size: i32| -> i32 { let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let Ok(msg) = mem.read_string_lossy(caller.as_context(), msg_ptr as u32, msg_size as u32) else { + return Status::InvalidMemoryAccess.as_i32(); + }; + let Some(lvl) = log_level_to_tracing(level) else { + return Status::BadArgument.as_i32(); }; - let msg = mem - .read_string_lossy(caller.as_context(), msg_ptr as u32, msg_size as u32) - .unwrap_or_default(); - let lvl = log_level_to_tracing(level); match lvl { tracing::Level::TRACE => tracing::trace!(target: "spacegate_plugin_wasm::guest", "{msg}"), tracing::Level::DEBUG => tracing::debug!(target: "spacegate_plugin_wasm::guest", "{msg}"), @@ -51,27 +77,45 @@ pub fn register_all( }, )?; - // ─────────── proxy_get_current_time_nanoseconds(return_time_ptr) ─────────── + linker.func_wrap( + "env", + "proxy_get_log_level", + |mut caller: Caller<'_, HostState>, return_ptr: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let lvl: LogLevel = host_max_log_level(); + if mem.write_u32(caller.as_context_mut(), return_ptr as u32, lvl.as_i32() as u32).is_err() { + return Status::InvalidMemoryAccess.as_i32(); + } + Status::Ok.as_i32() + }, + )?; + Ok(()) +} + +// ───────────────────────────────────────────────────────── +// Clocks / Timers / Context control（spec §Clocks §Timers §Context lifecycle） +// ───────────────────────────────────────────────────────── + +fn register_clock_and_tick(linker: &mut Linker) -> Result<(), wasmtime::Error> { linker.func_wrap( "env", "proxy_get_current_time_nanoseconds", |mut caller: Caller<'_, HostState>, return_ptr: i32| -> i32 { let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos() as u64) - .unwrap_or(0); + let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos() as u64).unwrap_or(0); if mem.write_u64(caller.as_context_mut(), return_ptr as u32, nanos).is_err() { - return Status::InternalFailure.as_i32(); + return Status::InvalidMemoryAccess.as_i32(); } Status::Ok.as_i32() }, )?; - // ─────────── proxy_set_tick_period_milliseconds(period) ─────────── linker.func_wrap( "env", "proxy_set_tick_period_milliseconds", @@ -80,52 +124,111 @@ pub fn register_all( Status::Ok.as_i32() }, )?; + Ok(()) +} - // ─────────── proxy_set_effective_context(context_id) ─────────── +fn register_context_control(linker: &mut Linker) -> Result<(), wasmtime::Error> { + // proxy_set_effective_context(context_id) -> Status linker.func_wrap( "env", "proxy_set_effective_context", |mut caller: Caller<'_, HostState>, ctx_id: i32| -> i32 { - caller.data_mut().effective_context = ctx_id as u32; - Status::Ok.as_i32() + let cid = ctx_id as u32; + let st = caller.data_mut(); + if st.contexts.contains_key(&cid) || cid == st.root_context_id { + st.effective_context = cid; + Status::Ok.as_i32() + } else { + Status::BadArgument.as_i32() + } }, )?; - // ─────────── proxy_done ─────────── - linker.func_wrap("env", "proxy_done", |_caller: Caller<'_, HostState>| -> i32 { Status::Ok.as_i32() })?; + // proxy_done() -> Status + // + // spec §proxy_done：guest 在 `proxy_on_done` 返回 false 之后调本 hostcall 表示「确实做完了」。 + // host 据此结束等待，进入 on_log/on_delete（在 vm.rs 处理）。 + linker.func_wrap("env", "proxy_done", |mut caller: Caller<'_, HostState>| -> i32 { + let st = caller.data_mut(); + let cid = st.effective_context; + if let Some(ctx) = st.contexts.get_mut(&cid) { + if !ctx.awaiting_done { + return Status::NotFound.as_i32(); + } + ctx.done_marker = true; + ctx.awaiting_done = false; + Status::Ok.as_i32() + } else { + Status::NotFound.as_i32() + } + })?; + Ok(()) +} - // ─────────── proxy_continue_stream(stream_type) ─────────── +// ───────────────────────────────────────────────────────── +// Stream control（spec §Common HTTP and TCP stream operations） +// ───────────────────────────────────────────────────────── + +fn register_stream_control(linker: &mut Linker) -> Result<(), wasmtime::Error> { + // proxy_continue_stream(stream_type) -> Status // - // hai 通过 `resume_http_request()` 调它，stream_type=0 表示 Request。 - // host 端把当前 ctx 的 continue_requested 置 true，Vm 状态机据此退出 await loop。 + // 我们 host 端仅处理 HTTP_REQUEST/HTTP_RESPONSE 的 continue：把当前 ctx 的 + // continue_requested 置 true，Vm 状态机据此退出 await loop。Downstream/Upstream + // 我们不接 TCP 层 → 返回 UNIMPLEMENTED（spec 允许）。 linker.func_wrap( "env", "proxy_continue_stream", |mut caller: Caller<'_, HostState>, stream_type: i32| -> i32 { - let st = caller.data(); - let ctx_id = st.effective_context; - let _ = StreamType::from_i32(stream_type); - if let Some(ctx) = caller.data_mut().contexts.get_mut(&ctx_id) { - ctx.continue_requested = true; + let Some(st_kind) = StreamType::from_i32(stream_type) else { + return Status::BadArgument.as_i32(); + }; + match st_kind { + StreamType::HttpRequest | StreamType::HttpResponse => { + let st = caller.data(); + let ctx_id = st.effective_context; + if let Some(ctx) = caller.data_mut().contexts.get_mut(&ctx_id) { + ctx.continue_requested = true; + } + Status::Ok.as_i32() + } + StreamType::Downstream | StreamType::Upstream => Status::Unimplemented.as_i32(), } - Status::Ok.as_i32() }, )?; - // ─────────── proxy_close_stream(stream_type) ─────────── + // proxy_close_stream(stream_type) -> Status linker.func_wrap( "env", "proxy_close_stream", - |_caller: Caller<'_, HostState>, _stream_type: i32| -> i32 { Status::Ok.as_i32() }, + |_caller: Caller<'_, HostState>, stream_type: i32| -> i32 { + match StreamType::from_i32(stream_type) { + Some(StreamType::HttpRequest) | Some(StreamType::HttpResponse) => Status::Ok.as_i32(), + Some(StreamType::Downstream) | Some(StreamType::Upstream) => Status::Unimplemented.as_i32(), + None => Status::BadArgument.as_i32(), + } + }, )?; + Ok(()) +} - // ─────────── proxy_get_buffer_bytes ─────────── - // - // 签名：(buffer_type, start, max_size, return_data_ptr, return_size_ptr) -> Status - // host 端要： - // 1. 从 HostState 拿对应 buffer（plugin_config / request_body / response_body / call_response_body） - // 2. 调 guest 的 `proxy_on_memory_allocate` 让它给一块缓冲 - // 3. 把字节写到 guest 内存，写回 *return_data = ptr, *return_size = len +// ───────────────────────────────────────────────────────── +// Buffers（spec §Buffers） +// ───────────────────────────────────────────────────────── + +/// 取 buffer 内容（克隆出一份，避免后续借用冲突）。 +fn read_buffer(state: &HostState, buf_type: BufferType) -> Option> { + match buf_type { + BufferType::PluginConfiguration | BufferType::VmConfiguration => Some(state.configuration.clone()), + BufferType::HttpRequestBody => state.current_context().and_then(|c| c.request_body.as_ref().map(|b| b.to_vec())), + BufferType::HttpResponseBody => state.current_context().and_then(|c| c.response_body.as_ref().map(|b| b.to_vec())), + BufferType::HttpCallResponseBody => state.current_context().map(|c| c.last_call_body.to_vec()), + // 未支持的（TCP / gRPC / FFI args）：buffer 类型本身合法，但当前 host 无数据 → NotFound + BufferType::DownstreamData | BufferType::UpstreamData | BufferType::GrpcCallMessage | BufferType::ForeignFunctionArguments => None, + } +} + +fn register_buffer(linker: &mut Linker) -> Result<(), wasmtime::Error> { + // proxy_get_buffer_bytes(buffer_type, start, max_size, *return_data, *return_size) -> Status linker.func_wrap( "env", "proxy_get_buffer_bytes", @@ -136,29 +239,13 @@ pub fn register_all( return_data_ptr: i32, return_size_ptr: i32| -> i32 { - let buf_type = BufferType::from_i32(buffer_type); - let bytes_opt: Option> = match buf_type { - BufferType::PluginConfiguration | BufferType::VmConfiguration => Some(caller.data().configuration.clone()), - BufferType::HttpRequestBody => caller - .data() - .current_context() - .and_then(|c| c.request_body.as_ref().map(|b| b.to_vec())), - BufferType::HttpResponseBody => caller - .data() - .current_context() - .and_then(|c| c.response_body.as_ref().map(|b| b.to_vec())), - BufferType::HttpCallResponseBody => caller - .data() - .current_context() - .map(|c| c.last_call_body.to_vec()), - BufferType::Unknown(_) => None, - }; - let bytes = match bytes_opt { - Some(b) => b, - None => return Status::NotFound.as_i32(), + let Some(buf_type) = BufferType::from_i32(buffer_type) else { + return Status::BadArgument.as_i32(); + }; + let bytes_opt = read_buffer(caller.data(), buf_type); + let Some(bytes) = bytes_opt else { + return Status::NotFound.as_i32(); }; - // 截取 [start, start + max_size)；max_size 是 u32 reinterpret 进来的， - // proxy-wasm-rust-sdk 经常传 usize::MAX -> u32::MAX，所以这里按 u32 重新解释。 let start = (start as u32) as usize; let max_size = (max_size as u32) as usize; if start > bytes.len() { @@ -166,73 +253,180 @@ pub fn register_all( } let end = (start.saturating_add(max_size)).min(bytes.len()); let slice = &bytes[start..end]; - // 空 buffer：写回 (0, 0) 并返回 Ok，让 guest 知道存在但是 0 长度 - if slice.is_empty() { - let mem = match MemoryHelper::from_caller(&mut caller) { - Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), - }; - let _ = mem.write_u32(caller.as_context_mut(), return_data_ptr as u32, 0); - let _ = mem.write_u32(caller.as_context_mut(), return_size_ptr as u32, 0); - return Status::Ok.as_i32(); + match write_alloc_pair(&mut caller, slice, return_data_ptr as u32, return_size_ptr as u32) { + Ok(()) => Status::Ok.as_i32(), + Err(s) => s.as_i32(), } - // 让 guest 分配 - let alloc = match caller.data().alloc.clone() { - Some(f) => f, - None => return Status::InternalFailure.as_i32(), + }, + )?; + + // proxy_get_buffer_status(buffer_type, *return_buffer_size, *return_unused) -> Status + linker.func_wrap( + "env", + "proxy_get_buffer_status", + |mut caller: Caller<'_, HostState>, buffer_type: i32, return_size_ptr: i32, return_unused_ptr: i32| -> i32 { + let Some(buf_type) = BufferType::from_i32(buffer_type) else { + return Status::BadArgument.as_i32(); }; - let guest_ptr = match alloc.call(&mut caller, slice.len() as u32) { - Ok(p) => p, - Err(_) => return Status::InternalFailure.as_i32(), + let len = match read_buffer(caller.data(), buf_type) { + Some(b) => b.len() as u32, + None => return Status::NotFound.as_i32(), }; let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; - if mem.write_bytes(caller.as_context_mut(), guest_ptr, slice).is_err() { - return Status::InternalFailure.as_i32(); + if mem.write_u32(caller.as_context_mut(), return_size_ptr as u32, len).is_err() { + return Status::InvalidMemoryAccess.as_i32(); } - let _ = mem.write_u32(caller.as_context_mut(), return_data_ptr as u32, guest_ptr); - let _ = mem.write_u32(caller.as_context_mut(), return_size_ptr as u32, slice.len() as u32); + let _ = mem.write_u32(caller.as_context_mut(), return_unused_ptr as u32, 0); Status::Ok.as_i32() }, )?; - // ─────────── proxy_set_buffer_bytes ─────────── + // proxy_set_buffer_bytes(buffer_type, start, size, *data, data_size) -> Status // - // 用于 guest 写回 response body（流式 hai 才会用，本阶段不实现完整流式，但 spec 要求接口存在）。 + // spec §Buffers proxy_set_buffer_bytes：可做 prepend / append / inject / replace。 + // start, size 解释为：用 (data, data_size) 替换 [start, start+size) 范围。 linker.func_wrap( "env", "proxy_set_buffer_bytes", |mut caller: Caller<'_, HostState>, buffer_type: i32, - _start: i32, - _size: i32, + start: i32, + size: i32, data_ptr: i32, data_size: i32| -> i32 { + let Some(buf_type) = BufferType::from_i32(buffer_type) else { + return Status::BadArgument.as_i32(); + }; let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; - let bytes = match mem.read_bytes(caller.as_context(), data_ptr as u32, data_size as u32) { + let new_bytes = match mem.read_bytes(caller.as_context(), data_ptr as u32, data_size as u32) { Ok(b) => b, - Err(_) => return Status::BadArgument.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; - let bt = BufferType::from_i32(buffer_type); let ctx_id = caller.data().effective_context; - if let Some(ctx) = caller.data_mut().contexts.get_mut(&ctx_id) { - match bt { - BufferType::HttpRequestBody => ctx.request_body = Some(Bytes::from(bytes)), - BufferType::HttpResponseBody => ctx.response_body = Some(Bytes::from(bytes)), - _ => return Status::BadArgument.as_i32(), + let st = caller.data_mut(); + let Some(ctx) = st.contexts.get_mut(&ctx_id) else { + return Status::NotFound.as_i32(); + }; + match buf_type { + BufferType::HttpRequestBody => { + let cur = ctx.request_body.take().unwrap_or_default(); + ctx.request_body = Some(splice_buffer(&cur, start as u32, size as u32, &new_bytes)); + Status::Ok.as_i32() + } + BufferType::HttpResponseBody => { + let cur = ctx.response_body.take().unwrap_or_default(); + ctx.response_body = Some(splice_buffer(&cur, start as u32, size as u32, &new_bytes)); + Status::Ok.as_i32() } + // TCP / gRPC / 配置 / FFI args：本 host 不支持写 + BufferType::DownstreamData + | BufferType::UpstreamData + | BufferType::GrpcCallMessage + | BufferType::VmConfiguration + | BufferType::PluginConfiguration + | BufferType::HttpCallResponseBody + | BufferType::ForeignFunctionArguments => Status::BadArgument.as_i32(), + } + }, + )?; + + Ok(()) +} + +/// spec §proxy_set_buffer_bytes：用 `replacement` 替换 `cur[start..start+size]`。 +fn splice_buffer(cur: &Bytes, start: u32, size: u32, replacement: &[u8]) -> Bytes { + let cur_len = cur.len(); + let start = (start as usize).min(cur_len); + let size = (size as usize).min(cur_len.saturating_sub(start)); + let mut out = Vec::with_capacity(cur_len.saturating_add(replacement.len())); + out.extend_from_slice(&cur[..start]); + out.extend_from_slice(replacement); + out.extend_from_slice(&cur[start + size..]); + Bytes::from(out) +} + +// ───────────────────────────────────────────────────────── +// HTTP fields（spec §HTTP fields） +// ───────────────────────────────────────────────────────── + +fn register_headers(linker: &mut Linker) -> Result<(), wasmtime::Error> { + // proxy_get_header_map_size(map_type, *return_size) -> Status + linker.func_wrap( + "env", + "proxy_get_header_map_size", + |mut caller: Caller<'_, HostState>, map_type: i32, return_size_ptr: i32| -> i32 { + let Some(mt) = MapType::from_i32(map_type) else { + return Status::BadArgument.as_i32(); + }; + let pairs = collect_pairs(caller.data(), mt); + let buf = { + let refs: Vec<(&[u8], &[u8])> = pairs.iter().map(|(k, v)| (k.as_slice(), v.as_slice())).collect(); + encode_pairs(&refs) + }; + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + if mem.write_u32(caller.as_context_mut(), return_size_ptr as u32, buf.len() as u32).is_err() { + return Status::InvalidMemoryAccess.as_i32(); + } + Status::Ok.as_i32() + }, + )?; + + // proxy_get_header_map_pairs + linker.func_wrap( + "env", + "proxy_get_header_map_pairs", + |mut caller: Caller<'_, HostState>, map_type: i32, return_data_ptr: i32, return_size_ptr: i32| -> i32 { + let Some(mt) = MapType::from_i32(map_type) else { + return Status::BadArgument.as_i32(); + }; + let pairs = collect_pairs(caller.data(), mt); + let buf = { + let refs: Vec<(&[u8], &[u8])> = pairs.iter().map(|(k, v)| (k.as_slice(), v.as_slice())).collect(); + encode_pairs(&refs) + }; + match write_alloc_pair(&mut caller, &buf, return_data_ptr as u32, return_size_ptr as u32) { + Ok(()) => Status::Ok.as_i32(), + Err(s) => s.as_i32(), } + }, + )?; + + // proxy_set_header_map_pairs + linker.func_wrap( + "env", + "proxy_set_header_map_pairs", + |mut caller: Caller<'_, HostState>, map_type: i32, data_ptr: i32, data_size: i32| -> i32 { + let Some(mt) = MapType::from_i32(map_type) else { + return Status::BadArgument.as_i32(); + }; + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let raw = match mem.read_bytes(caller.as_context(), data_ptr as u32, data_size as u32) { + Ok(b) => b, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let Some(pairs) = decode_pairs(&raw) else { + return Status::SerializationFailure.as_i32(); + }; + let new_map = pairs_to_header_map(&pairs); + replace_map(caller.data_mut(), mt, new_map); Status::Ok.as_i32() }, )?; - // ─────────── proxy_get_header_map_value ─────────── + // proxy_get_header_map_value linker.func_wrap( "env", "proxy_get_header_map_value", @@ -243,50 +437,30 @@ pub fn register_all( return_data_ptr: i32, return_size_ptr: i32| -> i32 { + let Some(mt) = MapType::from_i32(map_type) else { + return Status::BadArgument.as_i32(); + }; let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; let key = match mem.read_string_lossy(caller.as_context(), key_ptr as u32, key_size as u32) { Ok(s) => s, - Err(_) => return Status::BadArgument.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; let key_l = key.to_ascii_lowercase(); - let mt = MapType::from_i32(map_type); - let value_opt = lookup_header(caller.data(), mt, &key_l); - let Some(value) = value_opt else { + let Some(value) = lookup_header(caller.data(), mt, &key_l) else { return Status::NotFound.as_i32(); }; let bytes = value.into_bytes(); - // 空字符串也要分配 0 长度 - let alloc = match caller.data().alloc.clone() { - Some(f) => f, - None => return Status::InternalFailure.as_i32(), - }; - let guest_ptr = if bytes.is_empty() { - 0 - } else { - match alloc.call(&mut caller, bytes.len() as u32) { - Ok(p) => p, - Err(_) => return Status::InternalFailure.as_i32(), - } - }; - let mem = match MemoryHelper::from_caller(&mut caller) { - Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), - }; - if guest_ptr > 0 { - if mem.write_bytes(caller.as_context_mut(), guest_ptr, &bytes).is_err() { - return Status::InternalFailure.as_i32(); - } + match write_alloc_pair(&mut caller, &bytes, return_data_ptr as u32, return_size_ptr as u32) { + Ok(()) => Status::Ok.as_i32(), + Err(s) => s.as_i32(), } - let _ = mem.write_u32(caller.as_context_mut(), return_data_ptr as u32, guest_ptr); - let _ = mem.write_u32(caller.as_context_mut(), return_size_ptr as u32, bytes.len() as u32); - Status::Ok.as_i32() }, )?; - // ─────────── proxy_add_header_map_value ─────────── + // proxy_add_header_map_value linker.func_wrap( "env", "proxy_add_header_map_value", @@ -297,23 +471,26 @@ pub fn register_all( value_ptr: i32, value_size: i32| -> i32 { + let Some(mt) = MapType::from_i32(map_type) else { + return Status::BadArgument.as_i32(); + }; let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; let key = match mem.read_string_lossy(caller.as_context(), key_ptr as u32, key_size as u32) { Ok(s) => s, - Err(_) => return Status::BadArgument.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; let value = match mem.read_string_lossy(caller.as_context(), value_ptr as u32, value_size as u32) { Ok(s) => s, - Err(_) => return Status::BadArgument.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; - mutate_header(caller.data_mut(), MapType::from_i32(map_type), &key, HeaderMutation::Add(value)) + mutate_header(caller.data_mut(), mt, &key, HeaderMutation::Add(value)) }, )?; - // ─────────── proxy_replace_header_map_value ─────────── + // proxy_replace_header_map_value linker.func_wrap( "env", "proxy_replace_header_map_value", @@ -324,166 +501,88 @@ pub fn register_all( value_ptr: i32, value_size: i32| -> i32 { + let Some(mt) = MapType::from_i32(map_type) else { + return Status::BadArgument.as_i32(); + }; let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; let key = match mem.read_string_lossy(caller.as_context(), key_ptr as u32, key_size as u32) { Ok(s) => s, - Err(_) => return Status::BadArgument.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; let value = match mem.read_string_lossy(caller.as_context(), value_ptr as u32, value_size as u32) { Ok(s) => s, - Err(_) => return Status::BadArgument.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; - mutate_header(caller.data_mut(), MapType::from_i32(map_type), &key, HeaderMutation::Replace(value)) + mutate_header(caller.data_mut(), mt, &key, HeaderMutation::Replace(value)) }, )?; - // ─────────── proxy_remove_header_map_value ─────────── + // proxy_remove_header_map_value linker.func_wrap( "env", "proxy_remove_header_map_value", |mut caller: Caller<'_, HostState>, map_type: i32, key_ptr: i32, key_size: i32| -> i32 { + let Some(mt) = MapType::from_i32(map_type) else { + return Status::BadArgument.as_i32(); + }; let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; let key = match mem.read_string_lossy(caller.as_context(), key_ptr as u32, key_size as u32) { Ok(s) => s, - Err(_) => return Status::BadArgument.as_i32(), - }; - mutate_header(caller.data_mut(), MapType::from_i32(map_type), &key, HeaderMutation::Remove) - }, - )?; - - // ─────────── proxy_get_header_map_pairs ─────────── - linker.func_wrap( - "env", - "proxy_get_header_map_pairs", - |mut caller: Caller<'_, HostState>, map_type: i32, return_data_ptr: i32, return_size_ptr: i32| -> i32 { - let mt = MapType::from_i32(map_type); - let pairs = collect_pairs(caller.data(), mt); - let buf = { - let refs: Vec<(&[u8], &[u8])> = pairs.iter().map(|(k, v)| (k.as_slice(), v.as_slice())).collect(); - encode_pairs(&refs) + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; - let alloc = match caller.data().alloc.clone() { - Some(f) => f, - None => return Status::InternalFailure.as_i32(), - }; - let guest_ptr = match alloc.call(&mut caller, buf.len() as u32) { - Ok(p) => p, - Err(_) => return Status::InternalFailure.as_i32(), - }; - let mem = match MemoryHelper::from_caller(&mut caller) { - Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), - }; - if mem.write_bytes(caller.as_context_mut(), guest_ptr, &buf).is_err() { - return Status::InternalFailure.as_i32(); - } - let _ = mem.write_u32(caller.as_context_mut(), return_data_ptr as u32, guest_ptr); - let _ = mem.write_u32(caller.as_context_mut(), return_size_ptr as u32, buf.len() as u32); - Status::Ok.as_i32() + mutate_header(caller.data_mut(), mt, &key, HeaderMutation::Remove) }, )?; + Ok(()) +} - // ─────────── proxy_set_header_map_pairs ─────────── - linker.func_wrap( - "env", - "proxy_set_header_map_pairs", - |mut caller: Caller<'_, HostState>, map_type: i32, data_ptr: i32, data_size: i32| -> i32 { - let mem = match MemoryHelper::from_caller(&mut caller) { - Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), - }; - let raw = match mem.read_bytes(caller.as_context(), data_ptr as u32, data_size as u32) { - Ok(b) => b, - Err(_) => return Status::BadArgument.as_i32(), - }; - let Some(pairs) = decode_pairs(&raw) else { - return Status::BadArgument.as_i32(); - }; - let mt = MapType::from_i32(map_type); - let new_map = pairs_to_header_map(&pairs); - replace_map(caller.data_mut(), mt, new_map); - Status::Ok.as_i32() - }, - )?; +// ───────────────────────────────────────────────────────── +// Local response / status（spec §HTTP streams §proxy_send_local_response） +// ───────────────────────────────────────────────────────── - // ─────────── proxy_get_property ─────────── +fn register_status_and_local_response(linker: &mut Linker) -> Result<(), wasmtime::Error> { + // proxy_get_status(*return_status_code, **msg_data, *msg_size) -> Status // - // hai 用它读 `source.address`（客户端 IP）。我们把请求里能拿到的 source ip 提前 - // 放到 ctx.request_pseudo 或 properties 表里，host fn 这里检索。 + // spec §proxy_get_status：在 on_http_call_response 中返回该次 HTTP 调用的 status； + // 其它时机我们返回当前响应 status。 linker.func_wrap( "env", - "proxy_get_property", - |mut caller: Caller<'_, HostState>, path_ptr: i32, path_size: i32, return_data_ptr: i32, return_size_ptr: i32| -> i32 { - let mem = match MemoryHelper::from_caller(&mut caller) { - Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), - }; - let raw = match mem.read_bytes(caller.as_context(), path_ptr as u32, path_size as u32) { - Ok(b) => b, - Err(_) => return Status::BadArgument.as_i32(), - }; - // path 是用 '\0' 分割的多段（proxy-wasm 约定） - let segments: Vec<&[u8]> = raw.split(|b| *b == 0u8).filter(|s| !s.is_empty()).collect(); - // 我们暂时仅识别 `source.address` 一种（hai 唯一用例）。 - let value: Option> = if segments == [b"source".as_slice(), b"address".as_slice()] { - // 优先从 :authority / x-forwarded-for 推导（无客户端 socket 信息时降级） - caller - .data() - .current_context() - .and_then(|c| { - if !c.request_pseudo.authority.is_empty() { - Some(c.request_pseudo.authority.clone()) - } else { - None - } - }) - .map(|s| s.into_bytes()) - } else { - None - }; - let Some(bytes) = value else { - return Status::NotFound.as_i32(); - }; - let alloc = match caller.data().alloc.clone() { - Some(f) => f, - None => return Status::InternalFailure.as_i32(), - }; - let guest_ptr = match alloc.call(&mut caller, bytes.len() as u32) { - Ok(p) => p, - Err(_) => return Status::InternalFailure.as_i32(), + "proxy_get_status", + |mut caller: Caller<'_, HostState>, status_code_ptr: i32, msg_data_ptr: i32, msg_size_ptr: i32| -> i32 { + let (code, msg): (u32, String) = match caller.data().current_context() { + Some(c) => { + if c.last_call_status > 0 { + (c.last_call_status as u32, c.last_call_status_message.clone()) + } else if let Some(rs) = c.response_status { + (rs as u32, c.response_status_message.clone()) + } else { + (0, String::new()) + } + } + None => (0, String::new()), }; let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; - if mem.write_bytes(caller.as_context_mut(), guest_ptr, &bytes).is_err() { - return Status::InternalFailure.as_i32(); + if mem.write_u32(caller.as_context_mut(), status_code_ptr as u32, code).is_err() { + return Status::InvalidMemoryAccess.as_i32(); + } + let bytes = msg.into_bytes(); + match write_alloc_pair(&mut caller, &bytes, msg_data_ptr as u32, msg_size_ptr as u32) { + Ok(()) => Status::Ok.as_i32(), + Err(s) => s.as_i32(), } - let _ = mem.write_u32(caller.as_context_mut(), return_data_ptr as u32, guest_ptr); - let _ = mem.write_u32(caller.as_context_mut(), return_size_ptr as u32, bytes.len() as u32); - Status::Ok.as_i32() - }, - )?; - - // ─────────── proxy_set_property ───────────（stub） - linker.func_wrap( - "env", - "proxy_set_property", - |_caller: Caller<'_, HostState>, _p_ptr: i32, _p_size: i32, _v_ptr: i32, _v_size: i32| -> i32 { - Status::Ok.as_i32() }, )?; - // ─────────── proxy_send_local_response ─────────── - // - // 签名：(status, status_text_data, status_text_size, body_data, body_size, - // additional_headers_data, additional_headers_size, grpc_status) -> Status + // proxy_send_local_response(status, *status_text, status_text_size, *body, body_size, *headers, headers_size, grpc_status) linker.func_wrap( "env", "proxy_send_local_response", @@ -499,17 +598,21 @@ pub fn register_all( -> i32 { let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), + Err(_) => return Status::InvalidMemoryAccess.as_i32(), }; let body = if body_size > 0 { - mem.read_bytes(caller.as_context(), body_data as u32, body_size as u32) - .unwrap_or_default() + match mem.read_bytes(caller.as_context(), body_data as u32, body_size as u32) { + Ok(b) => b, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + } } else { Vec::new() }; let headers_bytes = if headers_size > 0 { - mem.read_bytes(caller.as_context(), headers_data as u32, headers_size as u32) - .unwrap_or_default() + match mem.read_bytes(caller.as_context(), headers_data as u32, headers_size as u32) { + Ok(b) => b, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + } } else { Vec::new() }; @@ -529,169 +632,619 @@ pub fn register_all( Status::Ok.as_i32() }, )?; + Ok(()) +} - // ─────────── proxy_http_call ─────────── - // - // 签名（type 15）： - // (upstream_data, upstream_size, headers_data, headers_size, - // body_data, body_size, trailers_data, trailers_size, timeout_ms, return_token_ptr) -> Status +// ───────────────────────────────────────────────────────── +// proxy_http_call（spec §HTTP calls） +// ───────────────────────────────────────────────────────── + +fn register_http_call(linker: &mut Linker, dispatch_tx: tokio::sync::mpsc::UnboundedSender<(u32, HttpCallResult)>) -> Result<(), wasmtime::Error> { linker.func_wrap( "env", "proxy_http_call", - { - let dispatch_tx = dispatch_tx.clone(); - move |mut caller: Caller<'_, HostState>, - upstream_data: i32, - upstream_size: i32, - headers_data: i32, - headers_size: i32, - body_data: i32, - body_size: i32, - _trailers_data: i32, - _trailers_size: i32, - timeout_ms: i32, - return_token_ptr: i32| - -> i32 { - let mem = match MemoryHelper::from_caller(&mut caller) { - Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), - }; - let cluster = match mem.read_string_lossy(caller.as_context(), upstream_data as u32, upstream_size as u32) { - Ok(s) => s, - Err(_) => return Status::BadArgument.as_i32(), - }; - let headers_bytes = mem.read_bytes(caller.as_context(), headers_data as u32, headers_size as u32).unwrap_or_default(); - let body = if body_size > 0 { - mem.read_bytes(caller.as_context(), body_data as u32, body_size as u32).unwrap_or_default() + move |mut caller: Caller<'_, HostState>, + upstream_data: i32, + upstream_size: i32, + headers_data: i32, + headers_size: i32, + body_data: i32, + body_size: i32, + _trailers_data: i32, + _trailers_size: i32, + timeout_ms: i32, + return_token_ptr: i32| + -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let cluster = match mem.read_string_lossy(caller.as_context(), upstream_data as u32, upstream_size as u32) { + Ok(s) => s, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let headers_bytes = mem.read_bytes(caller.as_context(), headers_data as u32, headers_size as u32).unwrap_or_default(); + let body = if body_size > 0 { + mem.read_bytes(caller.as_context(), body_data as u32, body_size as u32).unwrap_or_default() + } else { + Vec::new() + }; + let pairs = match decode_pairs(&headers_bytes) { + Some(p) => p, + None => return Status::SerializationFailure.as_i32(), + }; + let mut method = "GET".to_string(); + let mut path = "/".to_string(); + let mut authority = String::new(); + let mut others = Vec::with_capacity(pairs.len()); + for (k, v) in &pairs { + let key_str = String::from_utf8_lossy(k); + let val_str = String::from_utf8_lossy(v).into_owned(); + match key_str.as_ref() { + ":method" => method = val_str, + ":path" => path = val_str, + ":authority" => authority = val_str, + ":scheme" => {} + _ => others.push((key_str.to_string(), val_str)), + } + } + if method.is_empty() || path.is_empty() { + return Status::BadArgument.as_i32(); + } + let st = caller.data(); + let base = st.shell_cfg.resolve_cluster(&cluster).or_else(|| { + if !authority.is_empty() { + Some(format!("http://{authority}")) } else { - Vec::new() - }; - let pairs = decode_pairs(&headers_bytes).unwrap_or_default(); - // 解析 :method / :path / :authority - let mut method = "GET".to_string(); - let mut path = "/".to_string(); - let mut authority = String::new(); - let mut others = Vec::with_capacity(pairs.len()); - for (k, v) in &pairs { - let key_str = String::from_utf8_lossy(k); - let val_str = String::from_utf8_lossy(v).into_owned(); - match key_str.as_ref() { - ":method" => method = val_str, - ":path" => path = val_str, - ":authority" => authority = val_str, - ":scheme" => { /* host 层只用 http */ } - _ => others.push((key_str.to_string(), val_str)), - } + None } - // cluster → base URL - let st = caller.data(); - let base = st.shell_cfg.resolve_cluster(&cluster).or_else(|| { - if !authority.is_empty() { - Some(format!("http://{authority}")) - } else { - None + }); + let Some(base) = base else { + warn!(target: "spacegate_plugin_wasm", cluster = %cluster, "dispatch_http_call: cluster not configured"); + return Status::BadArgument.as_i32(); + }; + let url = format!("{}{}", base.trim_end_matches('/'), path); + let token = caller.data_mut().next_dispatch_token(); + let source_ctx = caller.data().effective_context; + caller.data_mut().pending_calls.insert( + token, + crate::host_state::PendingCall { + waker: None, + source_context_id: source_ctx, + }, + ); + let client = caller.data().http_client.clone(); + let timeout = Duration::from_millis(timeout_ms.max(1) as u64); + let tx = dispatch_tx.clone(); + tokio::spawn(async move { + debug!(target: "spacegate_plugin_wasm", %url, %method, "dispatch_http_call begin"); + let parsed_method = method.parse::().unwrap_or(reqwest::Method::GET); + let mut req = client.request(parsed_method, &url); + for (k, v) in others { + if k.starts_with(':') { + continue; } - }); - let Some(base) = base else { - warn!(target: "spacegate_plugin_wasm", cluster = %cluster, "dispatch_http_call: cluster not configured"); - return Status::BadArgument.as_i32(); - }; - let url = format!("{}{}", base.trim_end_matches('/'), path); - let token = caller.data_mut().next_dispatch_token(); - let source_ctx = caller.data().effective_context; - caller.data_mut().pending_calls.insert( - token, - crate::host_state::PendingCall { - waker: None, - source_context_id: source_ctx, - }, - ); - let client = caller.data().http_client.clone(); - let timeout = Duration::from_millis(timeout_ms.max(1) as u64); - let tx = dispatch_tx.clone(); - tokio::spawn(async move { - debug!(target: "spacegate_plugin_wasm", %url, %method, "dispatch_http_call begin"); - let parsed_method = match method.parse::() { - Ok(m) => m, - Err(_) => reqwest::Method::GET, - }; - let mut req = client.request(parsed_method, &url); - for (k, v) in others { - // 跳过 hop-by-hop / 非法字符头 - if k.starts_with(':') { - continue; - } - if let (Ok(name), Ok(val)) = (HeaderName::try_from(k.as_str()), HeaderValue::try_from(v.as_str())) { - req = req.header(name, val); - } + if let (Ok(name), Ok(val)) = (HeaderName::try_from(k.as_str()), HeaderValue::try_from(v.as_str())) { + req = req.header(name, val); } - if !body.is_empty() { - req = req.body(body); - } - req = req.timeout(timeout); - let result = match req.send().await { - Ok(resp) => { - let status = resp.status().as_u16(); - let mut hdrs = HeaderMap::new(); - for (k, v) in resp.headers().iter() { - if let (Ok(name), Ok(val)) = - (HeaderName::try_from(k.as_str()), HeaderValue::from_bytes(v.as_bytes())) - { - hdrs.append(name, val); - } - } - let body_bytes = resp.bytes().await.unwrap_or_default(); - HttpCallResult { - status, - headers: hdrs, - body: body_bytes, + } + if !body.is_empty() { + req = req.body(body); + } + req = req.timeout(timeout); + let result = match req.send().await { + Ok(resp) => { + let status = resp.status().as_u16(); + let status_message = resp.status().canonical_reason().unwrap_or("").to_string(); + let mut hdrs = HeaderMap::new(); + for (k, v) in resp.headers().iter() { + if let (Ok(name), Ok(val)) = (HeaderName::try_from(k.as_str()), HeaderValue::from_bytes(v.as_bytes())) { + hdrs.append(name, val); } } - Err(e) => { - warn!(target: "spacegate_plugin_wasm", %url, error = %e, "dispatch_http_call failed"); - HttpCallResult { - status: 0, - headers: HeaderMap::new(), - body: Bytes::new(), - } + let body_bytes = resp.bytes().await.unwrap_or_default(); + HttpCallResult { + status, + status_message, + headers: hdrs, + body: body_bytes, } - }; - debug!(target: "spacegate_plugin_wasm", token, status = result.status, "dispatch_http_call done"); - let _ = tx.send((token, result)); - }); - // 写回 token - let mem = match MemoryHelper::from_caller(&mut caller) { - Ok(m) => m, - Err(_) => return Status::InternalFailure.as_i32(), + } + Err(e) => { + warn!(target: "spacegate_plugin_wasm", %url, error = %e, "dispatch_http_call failed"); + HttpCallResult { + status: 0, + status_message: format!("{e}"), + headers: HeaderMap::new(), + body: Bytes::new(), + } + } }; - let _ = mem.write_u32(caller.as_context_mut(), return_token_ptr as u32, token); - info!(target: "spacegate_plugin_wasm", token, cluster = %cluster, "dispatch_http_call enqueued"); - Status::Ok.as_i32() + debug!(target: "spacegate_plugin_wasm", token, status = result.status, "dispatch_http_call done"); + let _ = tx.send((token, result)); + }); + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + if mem.write_u32(caller.as_context_mut(), return_token_ptr as u32, token).is_err() { + return Status::InvalidMemoryAccess.as_i32(); } + info!(target: "spacegate_plugin_wasm", token, cluster = %cluster, "dispatch_http_call enqueued"); + Status::Ok.as_i32() }, )?; + Ok(()) +} - // ─────────── 其余 hai-process-mix 模块声明但不用的 host fn，全部 stub 为 Ok ─────────── - // - // wasmtime 要求 instantiate 时所有 import 都已 link；这里返回 Ok 不影响功能。 - stub_all_unused(linker)?; +// ───────────────────────────────────────────────────────── +// Shared Data / Shared Queues（spec §Shared Key-Value Store §Shared Queues） +// ───────────────────────────────────────────────────────── + +fn register_shared_data_and_queue(linker: &mut Linker) -> Result<(), wasmtime::Error> { + // proxy_get_shared_data(*k, k_size, **v, *v_size, *cas) -> Status + linker.func_wrap( + "env", + "proxy_get_shared_data", + |mut caller: Caller<'_, HostState>, + k_ptr: i32, + k_size: i32, + v_data_ptr: i32, + v_size_ptr: i32, + cas_ptr: i32| + -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let key = match mem.read_bytes(caller.as_context(), k_ptr as u32, k_size as u32) { + Ok(b) => b, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let Some((value, cas)) = shared_data_get(&key) else { + return Status::NotFound.as_i32(); + }; + if let Err(s) = write_alloc_pair(&mut caller, &value, v_data_ptr as u32, v_size_ptr as u32) { + return s.as_i32(); + } + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + if mem.write_u32(caller.as_context_mut(), cas_ptr as u32, cas).is_err() { + return Status::InvalidMemoryAccess.as_i32(); + } + Status::Ok.as_i32() + }, + )?; + + // proxy_set_shared_data(*k, k_size, *v, v_size, cas) -> Status + linker.func_wrap( + "env", + "proxy_set_shared_data", + |mut caller: Caller<'_, HostState>, k_ptr: i32, k_size: i32, v_ptr: i32, v_size: i32, cas: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let key = match mem.read_bytes(caller.as_context(), k_ptr as u32, k_size as u32) { + Ok(b) => b, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let value = if v_size > 0 { + match mem.read_bytes(caller.as_context(), v_ptr as u32, v_size as u32) { + Ok(b) => b, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + } + } else { + Vec::new() + }; + match shared_data_set(&key, &value, cas as u32) { + SharedDataSetResult::Ok => Status::Ok.as_i32(), + SharedDataSetResult::CasMismatch => Status::CasMismatch.as_i32(), + } + }, + )?; + + // proxy_register_shared_queue(*n, n_size, *return_qid) -> Status + linker.func_wrap( + "env", + "proxy_register_shared_queue", + |mut caller: Caller<'_, HostState>, n_ptr: i32, n_size: i32, return_qid_ptr: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let name = match mem.read_string_lossy(caller.as_context(), n_ptr as u32, n_size as u32) { + Ok(s) => s, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let vm_id = caller.data().plugin_vm_id.clone(); + let qid = queue_register(&vm_id, &name); + if mem.write_u32(caller.as_context_mut(), return_qid_ptr as u32, qid).is_err() { + return Status::InvalidMemoryAccess.as_i32(); + } + Status::Ok.as_i32() + }, + )?; + + // proxy_resolve_shared_queue(*vm_id, vm_id_size, *n, n_size, *return_qid) -> Status + linker.func_wrap( + "env", + "proxy_resolve_shared_queue", + |mut caller: Caller<'_, HostState>, + vid_ptr: i32, + vid_size: i32, + n_ptr: i32, + n_size: i32, + return_qid_ptr: i32| + -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let vid = match mem.read_string_lossy(caller.as_context(), vid_ptr as u32, vid_size as u32) { + Ok(s) => s, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let name = match mem.read_string_lossy(caller.as_context(), n_ptr as u32, n_size as u32) { + Ok(s) => s, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let Some(qid) = queue_resolve(&vid, &name) else { + return Status::NotFound.as_i32(); + }; + if mem.write_u32(caller.as_context_mut(), return_qid_ptr as u32, qid).is_err() { + return Status::InvalidMemoryAccess.as_i32(); + } + Status::Ok.as_i32() + }, + )?; + + // proxy_enqueue_shared_queue(qid, *v, v_size) -> Status + linker.func_wrap( + "env", + "proxy_enqueue_shared_queue", + |mut caller: Caller<'_, HostState>, qid: i32, v_ptr: i32, v_size: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let bytes = if v_size > 0 { + match mem.read_bytes(caller.as_context(), v_ptr as u32, v_size as u32) { + Ok(b) => b, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + } + } else { + Vec::new() + }; + match queue_enqueue(qid as u32, &bytes) { + QueueOpResult::Ok => Status::Ok.as_i32(), + QueueOpResult::NotFound => Status::NotFound.as_i32(), + QueueOpResult::Empty => Status::Empty.as_i32(), + } + }, + )?; + + // proxy_dequeue_shared_queue(qid, **v, *v_size) -> Status + linker.func_wrap( + "env", + "proxy_dequeue_shared_queue", + |mut caller: Caller<'_, HostState>, qid: i32, v_data_ptr: i32, v_size_ptr: i32| -> i32 { + match queue_dequeue(qid as u32) { + (QueueOpResult::Ok, Some(bytes)) => match write_alloc_pair(&mut caller, &bytes, v_data_ptr as u32, v_size_ptr as u32) { + Ok(()) => Status::Ok.as_i32(), + Err(s) => s.as_i32(), + }, + (QueueOpResult::NotFound, _) => Status::NotFound.as_i32(), + _ => Status::Empty.as_i32(), + } + }, + )?; + + Ok(()) +} + +// ───────────────────────────────────────────────────────── +// Metrics（spec §Metrics） +// ───────────────────────────────────────────────────────── + +fn register_metrics(linker: &mut Linker) -> Result<(), wasmtime::Error> { + // proxy_define_metric(metric_type, *name, name_size, *return_mid) -> Status + linker.func_wrap( + "env", + "proxy_define_metric", + |mut caller: Caller<'_, HostState>, metric_type: i32, name_ptr: i32, name_size: i32, return_mid_ptr: i32| -> i32 { + let Some(kind) = MetricType::from_i32(metric_type) else { + return Status::BadArgument.as_i32(); + }; + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let name = match mem.read_string_lossy(caller.as_context(), name_ptr as u32, name_size as u32) { + Ok(s) => s, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let id = metric_define(kind, &name); + if mem.write_u32(caller.as_context_mut(), return_mid_ptr as u32, id).is_err() { + return Status::InvalidMemoryAccess.as_i32(); + } + Status::Ok.as_i32() + }, + )?; + + // proxy_record_metric(mid, value: u64) -> Status + linker.func_wrap( + "env", + "proxy_record_metric", + |_caller: Caller<'_, HostState>, mid: i32, value: i64| -> i32 { + match metric_record(mid as u32, value as u64) { + MetricOpResult::Ok => Status::Ok.as_i32(), + MetricOpResult::NotFound => Status::NotFound.as_i32(), + MetricOpResult::BadArgument => Status::BadArgument.as_i32(), + } + }, + )?; + + // proxy_increment_metric(mid, delta: i64) -> Status + linker.func_wrap( + "env", + "proxy_increment_metric", + |_caller: Caller<'_, HostState>, mid: i32, delta: i64| -> i32 { + match metric_increment(mid as u32, delta) { + MetricOpResult::Ok => Status::Ok.as_i32(), + MetricOpResult::NotFound => Status::NotFound.as_i32(), + MetricOpResult::BadArgument => Status::BadArgument.as_i32(), + } + }, + )?; + + // proxy_get_metric(mid, *return_value) -> Status + linker.func_wrap( + "env", + "proxy_get_metric", + |mut caller: Caller<'_, HostState>, mid: i32, return_ptr: i32| -> i32 { + let Some(v) = metric_get(mid as u32) else { + return Status::NotFound.as_i32(); + }; + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + if mem.write_u64(caller.as_context_mut(), return_ptr as u32, v).is_err() { + return Status::InvalidMemoryAccess.as_i32(); + } + Status::Ok.as_i32() + }, + )?; + Ok(()) +} + +// ───────────────────────────────────────────────────────── +// Properties（spec §Properties） +// ───────────────────────────────────────────────────────── + +fn register_property(linker: &mut Linker) -> Result<(), wasmtime::Error> { + // proxy_get_property(*path, path_size, **v, *v_size) -> Status + linker.func_wrap( + "env", + "proxy_get_property", + |mut caller: Caller<'_, HostState>, path_ptr: i32, path_size: i32, return_data_ptr: i32, return_size_ptr: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let raw = match mem.read_bytes(caller.as_context(), path_ptr as u32, path_size as u32) { + Ok(b) => b, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let segments = decode_property_path(&raw); + if segments.is_empty() { + return Status::NotFound.as_i32(); + } + // 1. 用户通过 proxy_set_property 写入的优先（spec 允许 host 自行决定） + let canonical_key = canonicalize_path(&segments); + if let Some(v) = caller.data().user_properties.get(&canonical_key).cloned() { + return match write_alloc_pair(&mut caller, &v, return_data_ptr as u32, return_size_ptr as u32) { + Ok(()) => Status::Ok.as_i32(), + Err(s) => s.as_i32(), + }; + } + // 2. well-known + let value = resolve_well_known(caller.data(), &segments); + let Some(value) = value else { + return Status::NotFound.as_i32(); + }; + match write_alloc_pair(&mut caller, &value, return_data_ptr as u32, return_size_ptr as u32) { + Ok(()) => Status::Ok.as_i32(), + Err(s) => s.as_i32(), + } + }, + )?; + + // proxy_set_property(*path, path_size, *v, v_size) -> Status + linker.func_wrap( + "env", + "proxy_set_property", + |mut caller: Caller<'_, HostState>, path_ptr: i32, path_size: i32, v_ptr: i32, v_size: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let raw_path = match mem.read_bytes(caller.as_context(), path_ptr as u32, path_size as u32) { + Ok(b) => b, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let segments = decode_property_path(&raw_path); + if segments.is_empty() { + return Status::BadArgument.as_i32(); + } + let canonical_key = canonicalize_path(&segments); + let value = if v_size > 0 { + match mem.read_bytes(caller.as_context(), v_ptr as u32, v_size as u32) { + Ok(b) => b, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + } + } else { + Vec::new() + }; + caller.data_mut().user_properties.insert(canonical_key, value); + Status::Ok.as_i32() + }, + )?; + Ok(()) +} + +fn canonicalize_path(segments: &[&[u8]]) -> Vec { + let mut out = Vec::new(); + for (i, s) in segments.iter().enumerate() { + if i > 0 { + out.push(0); + } + out.extend_from_slice(s); + } + out +} + +/// spec §Properties §Well-known properties：内置覆盖最常用的几个，其它返回 None。 +fn resolve_well_known(state: &HostState, segments: &[&[u8]]) -> Option> { + let path_str: Vec<&str> = segments.iter().filter_map(|s| std::str::from_utf8(s).ok()).collect(); + let joined = path_str.join("."); + match joined.as_str() { + // Proxy-Wasm + "plugin_name" => Some(state.plugin_name.as_bytes().to_vec()), + "plugin_root_id" => Some(state.plugin_root_id.as_bytes().to_vec()), + "plugin_vm_id" => Some(state.plugin_vm_id.as_bytes().to_vec()), + // Downstream connection + "source.address" => state.source_addr.map(|s| s.to_string().into_bytes()).or_else(|| { + // 退路：从 :authority 推导 + state.current_context().map(|c| c.request_pseudo.authority.clone().into_bytes()).filter(|b| !b.is_empty()) + }), + "source.port" => state.source_addr.map(|s| s.port().to_string().into_bytes()), + "destination.address" => state.destination_addr.map(|s| s.to_string().into_bytes()), + "destination.port" => state.destination_addr.map(|s| s.port().to_string().into_bytes()), + // HTTP request + "request.protocol" => state.current_context().map(|c| c.request_protocol.as_bytes().to_vec()).filter(|b| !b.is_empty()), + "request.size" => state.current_context().map(|c| c.request_size.to_string().into_bytes()), + "request.total_size" => state.current_context().map(|c| { + let hdr_bytes = approx_header_bytes(&c.request_headers); + (c.request_size + hdr_bytes as u64).to_string().into_bytes() + }), + // HTTP response + "response.size" => state.current_context().map(|c| c.response_size.to_string().into_bytes()), + "response.total_size" => state.current_context().map(|c| { + let hdr_bytes = approx_header_bytes(&c.response_headers); + (c.response_size + hdr_bytes as u64).to_string().into_bytes() + }), + _ => None, + } +} + +fn approx_header_bytes(map: &HeaderMap) -> usize { + let mut sum = 0; + for (k, v) in map.iter() { + sum += k.as_str().len() + 2 + v.as_bytes().len() + 2; + } + sum +} + +// ───────────────────────────────────────────────────────── +// gRPC（spec §gRPC calls）→ 全部返回 UNIMPLEMENTED +// ───────────────────────────────────────────────────────── + +fn register_grpc_unimplemented(linker: &mut Linker) -> Result<(), wasmtime::Error> { + linker.func_wrap( + "env", + "proxy_grpc_call", + |_caller: Caller<'_, HostState>, + _a: i32, + _b: i32, + _c: i32, + _d: i32, + _e: i32, + _f: i32, + _g: i32, + _h: i32, + _i: i32, + _j: i32, + _k: i32, + _l: i32| + -> i32 { Status::Unimplemented.as_i32() }, + )?; + linker.func_wrap( + "env", + "proxy_grpc_stream", + |_caller: Caller<'_, HostState>, + _a: i32, + _b: i32, + _c: i32, + _d: i32, + _e: i32, + _f: i32, + _g: i32, + _h: i32, + _i: i32| + -> i32 { Status::Unimplemented.as_i32() }, + )?; + linker.func_wrap("env", "proxy_grpc_cancel", |_caller: Caller<'_, HostState>, _t: i32| -> i32 { Status::Unimplemented.as_i32() })?; + linker.func_wrap("env", "proxy_grpc_close", |_caller: Caller<'_, HostState>, _t: i32| -> i32 { Status::Unimplemented.as_i32() })?; + linker.func_wrap( + "env", + "proxy_grpc_send", + |_caller: Caller<'_, HostState>, _t: i32, _m: i32, _ms: i32, _eos: i32| -> i32 { Status::Unimplemented.as_i32() }, + )?; + Ok(()) +} + +// ───────────────────────────────────────────────────────── +// Foreign function（spec §FFI）→ 没有注册表 → NotFound +// ───────────────────────────────────────────────────────── +fn register_foreign_function(linker: &mut Linker) -> Result<(), wasmtime::Error> { + linker.func_wrap( + "env", + "proxy_call_foreign_function", + |_caller: Caller<'_, HostState>, _a: i32, _b: i32, _c: i32, _d: i32, _e: i32, _f: i32| -> i32 { Status::NotFound.as_i32() }, + )?; Ok(()) } // ───────────────────────────────────────────────────────── -// 辅助：lookup / mutate / collect +// 辅助：alloc + 写 (data, size) pair；lookup / mutate / collect // ───────────────────────────────────────────────────────── +/// 在 guest 侧分配一段内存、写入 `payload`、然后把 (guest_ptr, len) 回写到 +/// `return_data_ptr` / `return_size_ptr`。 +/// +/// 空 payload：写 (0, 0)。 +fn write_alloc_pair(caller: &mut Caller<'_, HostState>, payload: &[u8], return_data_ptr: u32, return_size_ptr: u32) -> Result<(), Status> { + let mem = MemoryHelper::from_caller(caller).map_err(|_| Status::InvalidMemoryAccess)?; + if payload.is_empty() { + mem.write_u32(caller.as_context_mut(), return_data_ptr, 0).map_err(|_| Status::InvalidMemoryAccess)?; + mem.write_u32(caller.as_context_mut(), return_size_ptr, 0).map_err(|_| Status::InvalidMemoryAccess)?; + return Ok(()); + } + let alloc = caller.data().alloc.clone().ok_or(Status::InternalFailure)?; + let guest_ptr = alloc.call(&mut *caller, payload.len() as u32).map_err(|_| Status::InternalFailure)?; + let mem = MemoryHelper::from_caller(caller).map_err(|_| Status::InvalidMemoryAccess)?; + mem.write_bytes(caller.as_context_mut(), guest_ptr, payload).map_err(|_| Status::InvalidMemoryAccess)?; + mem.write_u32(caller.as_context_mut(), return_data_ptr, guest_ptr).map_err(|_| Status::InvalidMemoryAccess)?; + mem.write_u32(caller.as_context_mut(), return_size_ptr, payload.len() as u32).map_err(|_| Status::InvalidMemoryAccess)?; + Ok(()) +} + fn lookup_header(state: &HostState, mt: MapType, key_lower: &str) -> Option { let ctx = state.current_context()?; let map = match mt { - MapType::HttpRequestHeaders | MapType::HttpRequestTrailers => &ctx.request_headers, - MapType::HttpResponseHeaders | MapType::HttpResponseTrailers => &ctx.response_headers, - MapType::HttpCallResponseHeaders | MapType::HttpCallResponseTrailers => &ctx.last_call_headers, - MapType::Unknown(_) => return None, + MapType::HttpRequestHeaders => &ctx.request_headers, + MapType::HttpRequestTrailers => &ctx.request_trailers, + MapType::HttpResponseHeaders => &ctx.response_headers, + MapType::HttpResponseTrailers => &ctx.response_trailers, + MapType::HttpCallResponseHeaders => &ctx.last_call_headers, + MapType::HttpCallResponseTrailers => &ctx.last_call_trailers, + MapType::GrpcCallInitialMetadata | MapType::GrpcCallTrailingMetadata => return None, }; - // 伪头特判：:status / :method / :path / :authority / :scheme if let Some(value) = pseudo_lookup(ctx, mt, key_lower) { return Some(value); } @@ -725,11 +1278,13 @@ enum HeaderMutation { } fn mutate_header(state: &mut HostState, mt: MapType, key: &str, m: HeaderMutation) -> i32 { + if matches!(mt, MapType::GrpcCallInitialMetadata | MapType::GrpcCallTrailingMetadata) { + return Status::Unimplemented.as_i32(); + } let ctx_id = state.effective_context; let Some(ctx) = state.contexts.get_mut(&ctx_id) else { return Status::NotFound.as_i32(); }; - // 伪头处理（hai 不会改 :path 但理论上要支持） if key.starts_with(':') { let new_val = match &m { HeaderMutation::Add(v) | HeaderMutation::Replace(v) => Some(v.clone()), @@ -761,10 +1316,13 @@ fn mutate_header(state: &mut HostState, mt: MapType, key: &str, m: HeaderMutatio return Status::BadArgument.as_i32(); }; let map = match mt { - MapType::HttpRequestHeaders | MapType::HttpRequestTrailers => &mut ctx.request_headers, - MapType::HttpResponseHeaders | MapType::HttpResponseTrailers => &mut ctx.response_headers, - MapType::HttpCallResponseHeaders | MapType::HttpCallResponseTrailers => &mut ctx.last_call_headers, - MapType::Unknown(_) => return Status::BadArgument.as_i32(), + MapType::HttpRequestHeaders => &mut ctx.request_headers, + MapType::HttpRequestTrailers => &mut ctx.request_trailers, + MapType::HttpResponseHeaders => &mut ctx.response_headers, + MapType::HttpResponseTrailers => &mut ctx.response_trailers, + MapType::HttpCallResponseHeaders => &mut ctx.last_call_headers, + MapType::HttpCallResponseTrailers => &mut ctx.last_call_trailers, + MapType::GrpcCallInitialMetadata | MapType::GrpcCallTrailingMetadata => return Status::Unimplemented.as_i32(), }; match m { HeaderMutation::Add(v) => { @@ -789,13 +1347,15 @@ fn collect_pairs(state: &HostState, mt: MapType) -> Vec<(Vec, Vec)> { return Vec::new(); }; let map = match mt { - MapType::HttpRequestHeaders | MapType::HttpRequestTrailers => &ctx.request_headers, - MapType::HttpResponseHeaders | MapType::HttpResponseTrailers => &ctx.response_headers, - MapType::HttpCallResponseHeaders | MapType::HttpCallResponseTrailers => &ctx.last_call_headers, - MapType::Unknown(_) => return Vec::new(), + MapType::HttpRequestHeaders => &ctx.request_headers, + MapType::HttpRequestTrailers => &ctx.request_trailers, + MapType::HttpResponseHeaders => &ctx.response_headers, + MapType::HttpResponseTrailers => &ctx.response_trailers, + MapType::HttpCallResponseHeaders => &ctx.last_call_headers, + MapType::HttpCallResponseTrailers => &ctx.last_call_trailers, + MapType::GrpcCallInitialMetadata | MapType::GrpcCallTrailingMetadata => return Vec::new(), }; let mut out: Vec<(Vec, Vec)> = Vec::with_capacity(map.len() + 4); - // 加伪头 match mt { MapType::HttpRequestHeaders => { if !ctx.request_pseudo.method.is_empty() { @@ -850,121 +1410,9 @@ fn replace_map(state: &mut HostState, mt: MapType, new_map: HeaderMap) { }; match mt { MapType::HttpRequestHeaders => ctx.request_headers = new_map, + MapType::HttpRequestTrailers => ctx.request_trailers = new_map, MapType::HttpResponseHeaders => ctx.response_headers = new_map, + MapType::HttpResponseTrailers => ctx.response_trailers = new_map, _ => {} } } - -// ───────────────────────────────────────────────────────── -// stub：hai 模块声明但本阶段不需要语义的 host fn -// ───────────────────────────────────────────────────────── - -fn stub_all_unused(linker: &mut Linker) -> Result<(), wasmtime::Error> { - // proxy_get_shared_data / proxy_set_shared_data：暂返回 NotFound / Ok - linker.func_wrap( - "env", - "proxy_get_shared_data", - |_caller: Caller<'_, HostState>, _k_ptr: i32, _k_size: i32, _v_ptr: i32, _v_size: i32, _cas_ptr: i32| -> i32 { - Status::NotFound.as_i32() - }, - )?; - linker.func_wrap( - "env", - "proxy_set_shared_data", - |_caller: Caller<'_, HostState>, _k_ptr: i32, _k_size: i32, _v_ptr: i32, _v_size: i32, _cas: i32| -> i32 { - Status::Ok.as_i32() - }, - )?; - - // proxy_get_status：返回 Ok（与本地响应状态码相关，但 hai 不读取） - linker.func_wrap( - "env", - "proxy_get_status", - |_caller: Caller<'_, HostState>, _status_code_ptr: i32, _msg_ptr: i32, _msg_size: i32| -> i32 { Status::Ok.as_i32() }, - )?; - - // 共享队列 - linker.func_wrap( - "env", - "proxy_register_shared_queue", - |_caller: Caller<'_, HostState>, _n_ptr: i32, _n_size: i32, _ret: i32| -> i32 { Status::Empty.as_i32() }, - )?; - linker.func_wrap( - "env", - "proxy_resolve_shared_queue", - |_caller: Caller<'_, HostState>, _vid_ptr: i32, _vid_size: i32, _n_ptr: i32, _n_size: i32, _ret: i32| -> i32 { - Status::Empty.as_i32() - }, - )?; - linker.func_wrap( - "env", - "proxy_enqueue_shared_queue", - |_caller: Caller<'_, HostState>, _qid: i32, _v_ptr: i32, _v_size: i32| -> i32 { Status::Empty.as_i32() }, - )?; - linker.func_wrap( - "env", - "proxy_dequeue_shared_queue", - |_caller: Caller<'_, HostState>, _qid: i32, _v_ptr: i32, _v_size: i32| -> i32 { Status::Empty.as_i32() }, - )?; - - // gRPC 相关全部 Empty - linker.func_wrap( - "env", - "proxy_grpc_call", - |_caller: Caller<'_, HostState>, - _a: i32, - _b: i32, - _c: i32, - _d: i32, - _e: i32, - _f: i32, - _g: i32, - _h: i32, - _i: i32, - _j: i32, - _k: i32, - _l: i32| - -> i32 { Status::Empty.as_i32() }, - )?; - linker.func_wrap( - "env", - "proxy_grpc_stream", - |_caller: Caller<'_, HostState>, - _a: i32, - _b: i32, - _c: i32, - _d: i32, - _e: i32, - _f: i32, - _g: i32, - _h: i32, - _i: i32| - -> i32 { Status::Empty.as_i32() }, - )?; - linker.func_wrap( - "env", - "proxy_grpc_cancel", - |_caller: Caller<'_, HostState>, _t: i32| -> i32 { Status::Empty.as_i32() }, - )?; - linker.func_wrap( - "env", - "proxy_grpc_close", - |_caller: Caller<'_, HostState>, _t: i32| -> i32 { Status::Empty.as_i32() }, - )?; - linker.func_wrap( - "env", - "proxy_grpc_send", - |_caller: Caller<'_, HostState>, _t: i32, _m: i32, _ms: i32, _eos: i32| -> i32 { Status::Empty.as_i32() }, - )?; - - // foreign function：不支持 - linker.func_wrap( - "env", - "proxy_call_foreign_function", - |_caller: Caller<'_, HostState>, _a: i32, _b: i32, _c: i32, _d: i32, _e: i32, _f: i32| -> i32 { - Status::Empty.as_i32() - }, - )?; - - Ok(()) -} diff --git a/crates/plugin-wasm/src/host_state.rs b/crates/plugin-wasm/src/host_state.rs index f261151b..889c3677 100644 --- a/crates/plugin-wasm/src/host_state.rs +++ b/crates/plugin-wasm/src/host_state.rs @@ -1,13 +1,14 @@ //! 传给 `wasmtime::Store` 的宿主状态。 //! -//! - 顶层 `HostState` 承载：进程级 reqwest 客户端、shell 配置、序列化后的 plugin_config 字节、 +//! - 顶层 [`HostState`] 承载：进程级 reqwest 客户端、shell 配置、序列化后的 plugin_config 字节、 //! memory / 分配器 export、所有 HTTP 上下文、未完结的 `proxy_http_call` 句柄等。 //! - 每个 HTTP 请求建一个 [`RequestContext`]，由 `vm.rs` 在调 `proxy_on_*` 钩子前后维护。 //! - host fn 通过 `caller.data() / data_mut()` 读写 `HostState`，并以 -//! `effective_context` 字段定位「当前是哪个上下文」（hai-process-mix 调 -//! `proxy_set_effective_context` 切换）。 +//! `effective_context` 字段定位「当前是哪个上下文」（spec §Effective context changes， +//! guest 通过 `proxy_set_effective_context` 切换）。 use std::collections::HashMap; +use std::net::SocketAddr; use std::sync::Arc; use bytes::Bytes; @@ -26,8 +27,10 @@ pub enum ContextStage { Init, RequestHeaders, RequestBody, + RequestTrailers, ResponseHeaders, ResponseBody, + ResponseTrailers, Log, } @@ -53,6 +56,7 @@ pub struct LocalResponse { #[derive(Debug, Default)] pub struct HttpCallResult { pub status: u16, + pub status_message: String, pub headers: HeaderMap, pub body: Bytes, } @@ -73,20 +77,35 @@ pub struct RequestContext { pub stage: ContextStage, pub request_pseudo: PseudoHeaders, pub request_headers: HeaderMap, + pub request_trailers: HeaderMap, pub request_body: Option, pub response_status: Option, + pub response_status_message: String, pub response_headers: HeaderMap, + pub response_trailers: HeaderMap, pub response_body: Option, /// 上次 `proxy_http_call` 回调时由 host 注入；guest 通过 /// `get_http_call_response_*` 读它。 pub last_call_headers: HeaderMap, + pub last_call_trailers: HeaderMap, pub last_call_body: Bytes, /// 最近一次 dispatch_http_call 返回的状态码（hai 用 `:status` 伪头读取）。 pub last_call_status: u16, + pub last_call_status_message: String, /// guest 显式 `resume_http_request()` 后置 true；Vm 退出 Pause 等待循环。 pub continue_requested: bool, /// guest 调 `send_local_response` 后写入；Vm 据此短路返回。 pub local_response: Option, + /// HTTP 协议版本字符串（spec well-known property `request.protocol`）。 + pub request_protocol: String, + /// 收到的 request body 已知字节数。 + pub request_size: u64, + /// 输出的 response body 已知字节数。 + pub response_size: u64, + /// 通过 `proxy_done` 显式标记的 done 阶段（spec §proxy_done / §proxy_on_done）。 + pub done_marker: bool, + /// guest 上一次 `proxy_on_done` 返回值；false 表示要等 `proxy_done` 才能进 on_log/on_delete。 + pub awaiting_done: bool, } /// 进程内传给 wasmtime `Store` 的状态。生命周期与一次 Vm 实例一致。 @@ -98,14 +117,15 @@ pub struct HostState { pub configuration: Vec, /// guest 导出的线性内存（vm.rs 实例化完成后填）。 pub memory: Option, - /// guest 导出 `proxy_on_memory_allocate(size) -> ptr`：host 把数据回写到 wasm 前的分配函数。 + /// guest 导出 `proxy_on_memory_allocate(size) -> ptr` 或 deprecated `malloc(size) -> ptr`。 pub alloc: Option>, pub root_context_id: u32, /// 当前 hostcall 关联的上下文 id（由 vm.rs 在每次钩子前设置， /// 也可被 guest 的 `proxy_set_effective_context` 覆盖）。 pub effective_context: u32, pub contexts: HashMap, - /// guest 调用 `proxy_set_tick_period_milliseconds` 后存这里；Phase 1 暂不真正驱动 tick。 + /// guest 调用 `proxy_set_tick_period_milliseconds` 后存这里。 + /// `WasmPluginShell` 的后台 tick 任务 50ms 颗粒度地轮询本字段，到点 → `Vm::tick()`。 pub tick_period_ms: Option, /// 未完结的 dispatch_http_call 句柄表。 pub pending_calls: HashMap, @@ -113,16 +133,28 @@ pub struct HostState { next_token: u32, /// host 端 reqwest 客户端：所有 dispatch_http_call 复用一个，免去握手开销。 pub http_client: reqwest::Client, + /// 用户通过 `proxy_set_property` 设置的自定义属性（key = `\0` 分割的 path 字节）。 + pub user_properties: HashMap, Vec>, + /// 客户端 socket 地址（spec well-known property `source.address` / `source.port`）。 + pub source_addr: Option, + /// 服务端 socket 地址（spec well-known property `destination.address` / `destination.port`）。 + pub destination_addr: Option, + /// 插件标识（spec well-known property `plugin_name` / `plugin_root_id` / `plugin_vm_id`）。 + pub plugin_name: String, + pub plugin_root_id: String, + pub plugin_vm_id: String, } impl HostState { pub fn new(shell_cfg: Arc) -> Self { - // 把 plugin_config 序列化成 YAML 字节，与 hai-process-mix 的 `serde_yaml::from_slice` 对齐。 let configuration = shell_cfg.configuration_bytes(); let http_client = reqwest::Client::builder() .pool_max_idle_per_host(8) .build() .unwrap_or_else(|_| reqwest::Client::new()); + let plugin_name = shell_cfg.plugin_name.clone(); + let plugin_root_id = shell_cfg.plugin_root_id.clone(); + let plugin_vm_id = shell_cfg.plugin_vm_id.clone(); Self { shell_cfg, configuration, @@ -135,6 +167,12 @@ impl HostState { pending_calls: HashMap::new(), next_token: 1, http_client, + user_properties: HashMap::new(), + source_addr: None, + destination_addr: None, + plugin_name, + plugin_root_id, + plugin_vm_id, } } diff --git a/crates/plugin-wasm/src/lib.rs b/crates/plugin-wasm/src/lib.rs index b0906f01..e4db8e71 100644 --- a/crates/plugin-wasm/src/lib.rs +++ b/crates/plugin-wasm/src/lib.rs @@ -1,15 +1,65 @@ -//! SpaceGate **proxy-wasm（wasmtime）** 宿主 crate。 +//! SpaceGate **proxy-wasm (wasmtime) 宿主** crate。 //! -//! **集成方式**：不要从 `spacegate-plugin` 依赖本 crate（会形成循环依赖），应启用 `spacegate-shell` 的 `plugin-wasm` -//! feature；`spacegate_shell::startup` 会在网关启动时调用 [`register`]。 +//! **集成方式**：不要从 `spacegate-plugin` 依赖本 crate（会形成循环依赖），应启用 `spacegate-shell` +//! 的 `plugin-wasm` feature；`spacegate_shell::startup` 会在网关启动时调用 [`register`]。 //! -//! 当前实现度（对照 `spacegate演进方案-引入proxy-wasm.md` §4）： +//! # 与 [proxy-wasm/spec v0.2.1](https://github.com/proxy-wasm/spec) 的覆盖情况 //! -//! - ✅ [`WasmPluginShell`]：`Plugin::CODE = "wasm"`，`call` 真正驱动 wasm VM -//! - ✅ [`engine`] / [`runtime`]：进程级 wasmtime Engine + Module 缓存（按 url） -//! - ✅ [`vm::Vm`]：单 VM 异步状态机；驱动 `proxy_on_request_headers` → `proxy_on_http_call_response` → inner.call → `proxy_on_response_headers/body/log/done/delete` -//! - ✅ [`host_fn`]：proxy-wasm ABI 0.2.1 必要子集（log/time/header/buffer/property/local_response/dispatch_http_call/tick/continue/done） -//! - ⏳ 未做：VmPool（每请求新建 Vm）、ScanningBody（流式 SSE 截断）、fuel/epoch 资源隔离 +//! ## Host functions (env) +//! +//! - **Integration / Memory management**：guest 导出 `_initialize` 优先，否则回退 `_start`； +//! allocator 优先 `proxy_on_memory_allocate`，否则回退 `malloc`。 +//! - **Logging**：`proxy_log` / `proxy_get_log_level` 完整实现（host tracing 级别映射）。 +//! - **Clocks**：`proxy_get_current_time_nanoseconds` + `wasi_snapshot_preview1.clock_time_get`。 +//! - **Timers**：`proxy_set_tick_period_milliseconds` 完整生效；`shell.rs` 起一条 50ms 颗粒度的 +//! 后台 tokio 任务，到点 → `Vm::tick()` → guest `proxy_on_tick`。这要求 `Plugin::create` +//! 时存在 tokio runtime（spacegate-shell 的标准启动路径）；无 runtime 时降级为不驱动。 +//! - **Randomness**：`wasi_snapshot_preview1.random_get` 走 `getrandom`（OS RNG）。 +//! - **Environment**：`environ_*` 按 spec 全部返回 0/SUCCESS。 +//! - **Buffers**：`proxy_get_buffer_bytes` / `proxy_get_buffer_status` 覆盖 +//! HttpRequestBody / HttpResponseBody / HttpCallResponseBody / Vm/PluginConfiguration； +//! TCP / gRPC / FFI args 类型按 spec 返回 NotFound。 +//! `proxy_set_buffer_bytes` 实现 prepend / append / inject / replace 语义。 +//! - **HTTP fields**：`proxy_get_header_map_size/pairs/value` + add/replace/remove + set_pairs， +//! 覆盖 Request/Response/Trailers + HttpCallResponse Headers/Trailers；GRPC metadata 类型 +//! 按 spec 返回 Unimplemented。 +//! - **HTTP streams**：`proxy_send_local_response` / `proxy_continue_stream` / +//! `proxy_close_stream`（TCP downstream/upstream 按 spec 返回 Unimplemented）。 +//! - **HTTP calls**：`proxy_http_call`（reqwest 异步、`:method`/`:path`/`:authority` 校验， +//! 按 cluster map 或 `:authority` 兜底解析 URL）。 +//! - **Shared K/V**：`proxy_get/set_shared_data` 进程级 RwLock，含 CAS 比对。 +//! - **Shared queues**：`proxy_register/resolve/enqueue/dequeue_shared_queue` 进程级 Mutex VecDeque。 +//! - **Metrics**：`proxy_define/record/increment/get_metric` 进程级 Counter/Gauge/Histogram。 +//! - **Properties**：`proxy_get/set_property` 支持 well-known +//! (`plugin_name`/`plugin_root_id`/`plugin_vm_id`/`source.address`+`source.port`/ +//! `destination.address`+`destination.port`/`request.protocol`/`request.size`/`request.total_size`/ +//! `response.size`/`response.total_size`) 与用户自定义。 +//! - **gRPC**：按 spec 全部 `Unimplemented`。 +//! - **Foreign function**：按 spec `NotFound`（无注册表）。 +//! - **`proxy_done` / `proxy_set_effective_context`**：完整实现。 +//! +//! ## Guest callbacks driven by host +//! +//! - 启动：`_initialize`/`_start` → `proxy_on_context_create(root,0)` → +//! `proxy_on_vm_start` → `proxy_on_configure`（**仅一次**，由 `WasmPluginShell::create` 执行）。 +//! - 每请求：`proxy_on_context_create(http_id, root)` → `proxy_on_request_headers` → +//! （可选）`proxy_on_request_body` → （可选）`proxy_on_request_trailers` → +//! `inner.call` → `proxy_on_response_headers` → （可选）`proxy_on_response_body` → +//! （可选）`proxy_on_response_trailers` → `proxy_on_log` → `proxy_on_done` → `proxy_on_delete`。 +//! `WasmPluginShell` 持有 `Arc>`，所有请求串行经过同一 root VM +//! ——与 envoy/istio 的 per-worker 单线 wasm 模型一致。 +//! - 后台 `proxy_on_tick`：`shell.rs` 起 50ms 颗粒度的 tokio 任务驱动；guest 通过 +//! `proxy_set_tick_period_milliseconds` 改周期。 +//! - 异步 `proxy_on_http_call_response`：在 Pause 状态机里 await `dispatch_rx` 后回调。 +//! - Pause/Continue：`proxy_continue_stream` 同步解除 Pause；多次 dispatch 可串联。 +//! - Local response：`proxy_send_local_response` 任意 hook 都能短路。 +//! +//! ## 已知尚未驱动的回调（按设计取舍） +//! +//! - TCP 流回调（`proxy_on_new_connection`/`*_downstream_*`/`*_upstream_*`）： +//! spacegate-kernel 当前是 HTTP-only，TCP 插件层不支持。 +//! - `proxy_on_queue_ready` / `proxy_on_grpc_*` / `proxy_on_foreign_function`： +//! 对应 host fn 已为 spec 合规返回值；guest 侧回调不会被触发。 #![deny(clippy::unwrap_used, clippy::dbg_macro)] @@ -21,6 +71,7 @@ pub mod fetch; pub mod host_fn; pub mod host_state; pub mod runtime; +pub mod shared; pub mod shell; pub mod vm; diff --git a/crates/plugin-wasm/src/shared.rs b/crates/plugin-wasm/src/shared.rs new file mode 100644 index 00000000..9dbd4c73 --- /dev/null +++ b/crates/plugin-wasm/src/shared.rs @@ -0,0 +1,269 @@ +//! 进程级共享状态：spec §Shared Key-Value Store / §Shared Queues / §Metrics。 +//! +//! 这些设施按 proxy-wasm spec 必须在多个 VM / plugin 实例之间共享，因此放在进程级 `OnceCell` +//! + `RwLock` 之后；不依赖具体 `HostState`。 +//! +//! 实现要点： +//! - **Shared Data**：键值 + CAS（compare-and-swap）。每次成功 set 都使 cas 自增； +//! guest 传 `cas=0` 表示不校验。 +//! - **Shared Queues**：通过 `register_shared_queue(name)` / `resolve_shared_queue(vm_id, name)` 拿 qid； +//! `enqueue`/`dequeue` 操作 `VecDeque>`。 +//! - **Metrics**：Counter / Gauge / Histogram。Counter 不允许 decrement；Histogram 这里按 Gauge 处理 +//! （proxy-wasm 0.2.1 没有规定 histogram 的内部表示），足以满足 guest 的调用语义。 + +use std::collections::{HashMap, VecDeque}; +use std::sync::{Mutex, RwLock}; + +use once_cell::sync::Lazy; + +use crate::abi::MetricType; + +// ───────────────────────────────────────────────────────── +// Shared Data（spec §Shared Key-Value Store） +// ───────────────────────────────────────────────────────── + +#[derive(Debug, Default, Clone)] +pub struct SharedDataEntry { + pub value: Vec, + pub cas: u32, +} + +#[derive(Debug, Default)] +struct SharedDataStore { + map: HashMap, SharedDataEntry>, +} + +static SHARED_DATA: Lazy> = Lazy::new(|| RwLock::new(SharedDataStore::default())); + +/// 读：返回 (value, cas)；不存在返回 `None`。 +pub fn shared_data_get(key: &[u8]) -> Option<(Vec, u32)> { + let g = SHARED_DATA.read().ok()?; + g.map.get(key).map(|e| (e.value.clone(), e.cas)) +} + +#[derive(Debug, PartialEq, Eq)] +pub enum SharedDataSetResult { + Ok, + CasMismatch, +} + +/// 写：cas==0 表示不校验；非 0 必须等于当前 cas 才能成功。 +pub fn shared_data_set(key: &[u8], value: &[u8], cas: u32) -> SharedDataSetResult { + let Ok(mut g) = SHARED_DATA.write() else { + return SharedDataSetResult::CasMismatch; + }; + let entry = g.map.entry(key.to_vec()).or_default(); + if cas != 0 && cas != entry.cas { + return SharedDataSetResult::CasMismatch; + } + entry.value = value.to_vec(); + entry.cas = entry.cas.wrapping_add(1).max(1); + SharedDataSetResult::Ok +} + +// ───────────────────────────────────────────────────────── +// Shared Queues（spec §Shared Queues） +// ───────────────────────────────────────────────────────── + +#[derive(Debug, Default)] +struct SharedQueueRegistry { + by_id: HashMap>>, + by_name: HashMap<(String, String), u32>, // (vm_id, name) -> qid + next_id: u32, +} + +static SHARED_QUEUES: Lazy> = Lazy::new(|| Mutex::new(SharedQueueRegistry::default())); + +/// 注册（或打开已存在）一个共享队列；返回 qid。 +/// +/// `vm_id` 取本 VM 的 plugin_vm_id（按 spec 是 host 实现细节；这里用 "default"）。 +pub fn queue_register(vm_id: &str, name: &str) -> u32 { + let mut g = match SHARED_QUEUES.lock() { + Ok(g) => g, + Err(p) => p.into_inner(), + }; + let key = (vm_id.to_string(), name.to_string()); + if let Some(qid) = g.by_name.get(&key).copied() { + return qid; + } + g.next_id = g.next_id.wrapping_add(1).max(1); + let qid = g.next_id; + g.by_id.insert(qid, VecDeque::new()); + g.by_name.insert(key, qid); + qid +} + +/// 解析已存在的队列；不存在返回 None。 +pub fn queue_resolve(vm_id: &str, name: &str) -> Option { + let g = SHARED_QUEUES.lock().ok()?; + g.by_name.get(&(vm_id.to_string(), name.to_string())).copied() +} + +#[derive(Debug, PartialEq, Eq)] +pub enum QueueOpResult { + Ok, + NotFound, + Empty, +} + +pub fn queue_enqueue(qid: u32, value: &[u8]) -> QueueOpResult { + let mut g = match SHARED_QUEUES.lock() { + Ok(g) => g, + Err(p) => p.into_inner(), + }; + match g.by_id.get_mut(&qid) { + Some(q) => { + q.push_back(value.to_vec()); + QueueOpResult::Ok + } + None => QueueOpResult::NotFound, + } +} + +pub fn queue_dequeue(qid: u32) -> (QueueOpResult, Option>) { + let mut g = match SHARED_QUEUES.lock() { + Ok(g) => g, + Err(p) => p.into_inner(), + }; + match g.by_id.get_mut(&qid) { + Some(q) => match q.pop_front() { + Some(v) => (QueueOpResult::Ok, Some(v)), + None => (QueueOpResult::Empty, None), + }, + None => (QueueOpResult::NotFound, None), + } +} + +// ───────────────────────────────────────────────────────── +// Metrics（spec §Metrics） +// ───────────────────────────────────────────────────────── + +#[derive(Debug)] +struct MetricEntry { + kind: MetricType, + value: u64, + #[allow(dead_code)] + name: String, +} + +#[derive(Debug, Default)] +struct MetricRegistry { + by_id: HashMap, + by_name: HashMap, + next_id: u32, +} + +static METRICS: Lazy> = Lazy::new(|| Mutex::new(MetricRegistry::default())); + +#[derive(Debug, PartialEq, Eq)] +pub enum MetricOpResult { + Ok, + NotFound, + BadArgument, +} + +pub fn metric_define(kind: MetricType, name: &str) -> u32 { + let mut g = match METRICS.lock() { + Ok(g) => g, + Err(p) => p.into_inner(), + }; + if let Some(id) = g.by_name.get(name).copied() { + return id; + } + g.next_id = g.next_id.wrapping_add(1).max(1); + let id = g.next_id; + g.by_id.insert(id, MetricEntry { kind, value: 0, name: name.to_string() }); + g.by_name.insert(name.to_string(), id); + id +} + +pub fn metric_record(id: u32, value: u64) -> MetricOpResult { + let mut g = match METRICS.lock() { + Ok(g) => g, + Err(p) => p.into_inner(), + }; + match g.by_id.get_mut(&id) { + Some(m) => { + m.value = value; + MetricOpResult::Ok + } + None => MetricOpResult::NotFound, + } +} + +pub fn metric_increment(id: u32, delta: i64) -> MetricOpResult { + let mut g = match METRICS.lock() { + Ok(g) => g, + Err(p) => p.into_inner(), + }; + let Some(m) = g.by_id.get_mut(&id) else { + return MetricOpResult::NotFound; + }; + if matches!(m.kind, MetricType::Counter) && delta < 0 { + return MetricOpResult::BadArgument; + } + if delta >= 0 { + m.value = m.value.saturating_add(delta as u64); + } else { + m.value = m.value.saturating_sub((-delta) as u64); + } + MetricOpResult::Ok +} + +pub fn metric_get(id: u32) -> Option { + let g = METRICS.lock().ok()?; + g.by_id.get(&id).map(|m| m.value) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shared_data_cas_roundtrip() { + let key = b"shared_data_cas_roundtrip_key"; + assert_eq!(shared_data_set(key, b"v1", 0), SharedDataSetResult::Ok); + let (v, cas1) = shared_data_get(key).unwrap(); + assert_eq!(v, b"v1"); + assert!(cas1 > 0); + assert_eq!(shared_data_set(key, b"v2", 99), SharedDataSetResult::CasMismatch); + assert_eq!(shared_data_set(key, b"v2", cas1), SharedDataSetResult::Ok); + let (v, cas2) = shared_data_get(key).unwrap(); + assert_eq!(v, b"v2"); + assert!(cas2 > cas1); + } + + #[test] + fn shared_queue_roundtrip() { + let qid = queue_register("default", "shared_queue_roundtrip_q"); + assert_eq!(queue_enqueue(qid, b"a"), QueueOpResult::Ok); + assert_eq!(queue_enqueue(qid, b"b"), QueueOpResult::Ok); + let (s, v) = queue_dequeue(qid); + assert_eq!(s, QueueOpResult::Ok); + assert_eq!(v.as_deref(), Some(b"a".as_slice())); + let (s, v) = queue_dequeue(qid); + assert_eq!(s, QueueOpResult::Ok); + assert_eq!(v.as_deref(), Some(b"b".as_slice())); + let (s, _) = queue_dequeue(qid); + assert_eq!(s, QueueOpResult::Empty); + } + + #[test] + fn metric_counter_increment_only() { + let id = metric_define(MetricType::Counter, "metric_counter_increment_only"); + assert_eq!(metric_increment(id, 3), MetricOpResult::Ok); + assert_eq!(metric_get(id), Some(3)); + assert_eq!(metric_increment(id, -1), MetricOpResult::BadArgument); + assert_eq!(metric_get(id), Some(3)); + } + + #[test] + fn metric_gauge_bidirectional() { + let id = metric_define(MetricType::Gauge, "metric_gauge_bidirectional"); + assert_eq!(metric_increment(id, 5), MetricOpResult::Ok); + assert_eq!(metric_increment(id, -2), MetricOpResult::Ok); + assert_eq!(metric_get(id), Some(3)); + assert_eq!(metric_record(id, 100), MetricOpResult::Ok); + assert_eq!(metric_get(id), Some(100)); + } +} diff --git a/crates/plugin-wasm/src/shell.rs b/crates/plugin-wasm/src/shell.rs index 924db1f4..3251fffa 100644 --- a/crates/plugin-wasm/src/shell.rs +++ b/crates/plugin-wasm/src/shell.rs @@ -1,21 +1,42 @@ -//! `Plugin` 实现：在 `call` 内异步驱动 wasm Vm，按 fail_strategy 处理 Trap。 +//! `Plugin` 实现：实例化一次长生命 Vm，后续请求复用，并起一条后台 tick 任务驱动 `proxy_on_tick`。 //! -//! 首版未做 VM 池：每次请求新建 Vm。hai-process-mix 实例化 + 配置一遍约几毫秒； -//! 后续按演进文档 §4.3 接 `VmPool` 即可显著降损。 +//! 与旧版「每请求新建 Vm」相比的取舍： +//! +//! - 优点：guest 的 root context 可保留状态；`proxy_on_tick` 可真正按 `proxy_set_tick_period_milliseconds` 周期触发； +//! `on_vm_start` / `on_configure` 仅跑一次，热路径少几毫秒。 +//! - 代价：所有经过本插件实例的请求会通过同一把 `tokio::sync::Mutex` 串行化处理—— +//! wasmtime `Store` 是 !Sync，无法并发；envoy / istio 的 proxy-wasm 实现也是相同模型。 +//! +//! 后续要做更细粒度并发（多 root VM 池）属于演进文档 §4.3 范畴，本版不在范围内。 use std::sync::Arc; +use std::time::{Duration, Instant}; use spacegate_kernel::{SgBody, SgRequest, SgResponse}; use spacegate_plugin::{BoxError, Inner, Plugin, PluginConfig}; +use tokio::sync::Mutex as AsyncMutex; use crate::config::{FailStrategy, WasmPluginShellConfig}; use crate::runtime::default_module_cache; use crate::vm::Vm; +/// Drop 时 abort 关联的 tokio 任务；保证后台 tick 不会在 shell 析构后继续持有 Vm 引用。 +struct AbortOnDrop(tokio::task::JoinHandle<()>); +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.abort(); + } +} + /// Proxy-Wasm 宿主壳插件（`CODE = "wasm"`）。 pub struct WasmPluginShell { cfg: Arc, + #[allow(dead_code)] module: Arc, + vm: Arc>, + /// 后台 tick 任务句柄；shell drop 时自动 abort。 + /// `None` 表示创建时没有 tokio runtime 上下文（非测试常见路径），tick 退化为不驱动。 + _tick_task: Option, } impl Plugin for WasmPluginShell { @@ -23,7 +44,7 @@ impl Plugin for WasmPluginShell { fn call(&self, req: SgRequest, inner: Inner) -> impl std::future::Future> + Send { let cfg = self.cfg.clone(); - let module = self.module.clone(); + let vm = self.vm.clone(); async move { tracing::info!( target: "spacegate_plugin_wasm", @@ -31,16 +52,8 @@ impl Plugin for WasmPluginShell { uri = %req.uri(), "wasm plugin shell: request entered plugin layer" ); - let vm_res = Vm::new(&module, cfg.clone()).await; - let mut vm = match vm_res { - Ok(v) => v, - Err(e) => { - tracing::error!(target: "spacegate_plugin_wasm", error = %e, "Vm::new failed"); - return Ok(passthrough_on_error(e.to_string(), req, inner, cfg.fail_strategy).await); - } - }; - tracing::info!(target: "spacegate_plugin_wasm", "Vm initialized, entering process"); - match vm.process(req, inner).await { + let mut guard = vm.lock().await; + match guard.process(req, inner).await { Ok(resp) => { tracing::info!(target: "spacegate_plugin_wasm", status = %resp.status(), "Vm::process ok"); Ok(resp) @@ -77,20 +90,59 @@ impl Plugin for WasmPluginShell { ); let cache = default_module_cache(); let module = cache.get_or_compile(cfg.url.trim()).map_err(|e| -> BoxError { format!("compile wasm: {e}").into() })?; + let cfg = Arc::new(cfg); + let vm = Vm::new(&module, cfg.clone()).map_err(|e| -> BoxError { format!("Vm::new: {e}").into() })?; + let vm = Arc::new(AsyncMutex::new(vm)); + let tick_task = spawn_tick_loop(&vm); Ok(Self { - cfg: Arc::new(cfg), + cfg, module, + vm, + _tick_task: tick_task, }) } } -/// 当 Vm::new 失败时，原 `req` 已经被消费；按 fail_strategy 合成一个最简单的兜底响应。 -async fn passthrough_on_error(err: String, _req: SgRequest, _inner: Inner, fs: FailStrategy) -> SgResponse { - let status = match fs { - FailStrategy::FailOpen => http::StatusCode::BAD_GATEWAY, - FailStrategy::FailClose => http::StatusCode::INTERNAL_SERVER_ERROR, - }; - let mut resp = SgResponse::new(SgBody::full(format!("wasm plugin init failed: {err}"))); - *resp.status_mut() = status; - resp +/// 起一条 50ms 粒度的轮询任务：每个 tick 看一眼 `Vm::tick_period_ms()`，到点了就 `Vm::tick()`。 +/// +/// - 粒度 50ms 是工程取舍：spec 没有规定 tick 必须精确，envoy 也是大颗粒度；如果 guest 设置 < 50ms 的周期， +/// 实际触发率会被压到 50ms 一次——记入 `lib.rs` 顶部已知限制。 +/// - 任务持有 `Arc>`，shell drop 时 `AbortOnDrop` 立刻 abort，不存在悬挂任务。 +/// - 若 `proxy_on_tick` trap，记 error 后退出循环（防止热循环 panic）。 +fn spawn_tick_loop(vm: &Arc>) -> Option { + let handle = tokio::runtime::Handle::try_current().ok()?; + let vm = vm.clone(); + let task = handle.spawn(async move { + const POLL_GRANULARITY: Duration = Duration::from_millis(50); + let mut interval = tokio::time::interval(POLL_GRANULARITY); + // 首次 tick 立刻就绪——跳过它，避免一启动就触发 on_tick。 + interval.tick().await; + let mut last_tick: Option = None; + loop { + interval.tick().await; + let mut guard = vm.lock().await; + let period = guard.tick_period_ms(); + if period == 0 { + last_tick = None; + continue; + } + let due = match last_tick { + Some(t) => t.elapsed().as_millis() as u64 >= period as u64, + None => true, + }; + if !due { + continue; + } + if let Err(e) = guard.tick() { + tracing::error!( + target: "spacegate_plugin_wasm", + error = %e, + "proxy_on_tick failed; stopping tick task" + ); + return; + } + last_tick = Some(Instant::now()); + } + }); + Some(AbortOnDrop(task)) } diff --git a/crates/plugin-wasm/src/vm.rs b/crates/plugin-wasm/src/vm.rs index 8435c775..d0e6ff46 100644 --- a/crates/plugin-wasm/src/vm.rs +++ b/crates/plugin-wasm/src/vm.rs @@ -3,37 +3,42 @@ //! 一个 `Vm` 包含： //! //! - `wasmtime::Store`：宿主状态 + linear memory -//! - `wasmtime::Instance`：实例化后的 hai-process-mix +//! - `wasmtime::Instance`：实例化后的 wasm guest //! - 缓存的 guest exports（避免每次按名查找） //! //! 关键流程（[`Vm::process`]）： //! //! 1. `proxy_on_context_create(http_ctx_id, root_id)` -//! 2. `proxy_on_request_headers` → 解析 Action +//! 2. `proxy_on_request_headers` → 解析 Action（end_of_stream=false 当有 body 时） //! 3. 若 Pause → `drive_until_continue`：循环 await dispatch_http_call 结果 → -//! 调 `proxy_on_http_call_response` → 直至 guest `continue_stream(Request)` 或 `send_local_response` -//! 4. 若 guest 写了 `local_response`：直接返回它（短路 inner.call） -//! 5. 否则：把 ctx 内的 headers 同步回 `SgRequest`，调 `inner.call` -//! 6. `proxy_on_response_headers` / `proxy_on_response_body` / `proxy_on_log` -//! 7. ctx 清理；vm 归池 +//! 调 `proxy_on_http_call_response` → 直至 guest `continue_stream(HTTP_REQUEST)` 或 `send_local_response` +//! 4. 若 guest 导出 `proxy_on_request_body`：收齐 body，调 hook，可能再次 Pause；body 写回 SgRequest +//! 5. 若 guest 导出 `proxy_on_request_trailers`：用空 trailer map 调一次（spacegate 暂不暴露 trailer） +//! 6. 若 guest 写了 `local_response`：直接返回它（短路 inner.call） +//! 7. 否则：把 ctx 内的 headers/body 同步回 `SgRequest`，调 `inner.call` +//! 8. `proxy_on_response_headers` / `proxy_on_response_body` / `proxy_on_response_trailers` / `proxy_on_log` / +//! `proxy_on_done` (spec 要求 false 时等 `proxy_done`) / `proxy_on_delete` +//! 9. ctx 清理 use std::sync::Arc; use bytes::Bytes; -use http::{HeaderMap, HeaderValue}; +use http::HeaderMap; use http_body_util::BodyExt; use spacegate_kernel::{SgBody, SgRequest, SgResponse}; use tracing::{debug, info, warn}; -use wasmtime::{AsContextMut, Instance, Linker, Store, TypedFunc}; +use wasmtime::{AsContext, AsContextMut, Instance, Linker, Store, TypedFunc}; -use crate::abi::Action; +use crate::abi::{Action, MemoryHelper}; use crate::config::{FailStrategy, WasmPluginShellConfig}; use crate::engine::shared_engine; use crate::error::WasmHostError; use crate::host_fn::register_all; use crate::host_state::{ContextStage, HostState, HttpCallResult, PseudoHeaders, RequestContext}; -/// 一次性 Vm（每次 plugin 调用都新建；首版不做池）。 +/// 长生命 Vm：插件 `create` 时实例化一次，之后被多次请求复用，再加一条 +/// 后台 tick 任务用来驱动 `proxy_on_tick`。`store` !Sync，所以共享时必须 +/// 套 `tokio::sync::Mutex`（见 `shell.rs`）。 pub struct Vm { store: Store, #[allow(dead_code)] @@ -46,17 +51,25 @@ pub struct Vm { fn_on_vm_start: Option>, fn_on_configure: TypedFunc<(u32, u32), u32>, fn_on_request_headers: TypedFunc<(u32, u32, u32), u32>, + fn_on_request_body: Option>, + fn_on_request_trailers: Option>, fn_on_response_headers: TypedFunc<(u32, u32, u32), u32>, fn_on_response_body: Option>, + fn_on_response_trailers: Option>, fn_on_http_call_response: TypedFunc<(u32, u32, u32, u32, u32), ()>, fn_on_log: Option>, fn_on_done: Option>, fn_on_delete: Option>, + fn_on_tick: Option>, } impl Vm { /// 创建并启动一个 Vm：实例化 → 缓存 exports → 跑 vm_start/configure。 - pub async fn new(module: &wasmtime::Module, shell_cfg: Arc) -> Result { + /// + /// 这是同步函数。整个过程不涉及 `await`（wasmtime 编译/实例化、guest `_initialize`、 + /// `on_vm_start` / `on_configure` 全部是同步调用），所以 `WasmPluginShell::create` + /// 这种 sync 上下文也能直接构造。 + pub fn new(module: &wasmtime::Module, shell_cfg: Arc) -> Result { let engine = shared_engine(); let host = HostState::new(shell_cfg.clone()); let mut store: Store = Store::new(engine, host); @@ -64,32 +77,34 @@ impl Vm { let (dispatch_tx, dispatch_rx) = tokio::sync::mpsc::unbounded_channel::<(u32, HttpCallResult)>(); register_all(&mut linker, dispatch_tx).map_err(|e| WasmHostError::Instantiate(format!("register host fn: {e}")))?; - // hai_process_mix 是 wasi reactor（_initialize export），但它 imports - // `wasi_snapshot_preview1` 的 environ_get / fd_write / proc_exit / random_get - // 等。我们用占位实现，给基础语义即可——hai 实际只在 log/random/clock 处依赖。 register_wasi_stubs(&mut linker)?; let instance = linker .instantiate(&mut store, module) .map_err(|e| WasmHostError::Instantiate(format!("instantiate: {e}")))?; - // 拿 memory + alloc let memory = instance .get_memory(&mut store, "memory") .ok_or_else(|| WasmHostError::AbiViolation("no `memory` export".into()))?; store.data_mut().memory = Some(memory); + // spec §Memory management：优先 `proxy_on_memory_allocate`，否则回退 `malloc`。 if let Ok(alloc) = instance.get_typed_func::(&mut store, "proxy_on_memory_allocate") { store.data_mut().alloc = Some(alloc); + } else if let Ok(alloc) = instance.get_typed_func::(&mut store, "malloc") { + store.data_mut().alloc = Some(alloc); } else { - return Err(WasmHostError::AbiViolation("no `proxy_on_memory_allocate` export".into())); + return Err(WasmHostError::AbiViolation( + "no memory allocator export (proxy_on_memory_allocate or malloc)".into(), + )); } - // 先跑 `_initialize`（wasi reactor） + // spec §Integration：先 `_initialize`；若不存在尝试 `_start`。 if let Ok(init) = instance.get_typed_func::<(), ()>(&mut store, "_initialize") { init.call(&mut store, ()).map_err(|e| WasmHostError::Instantiate(format!("_initialize: {e}")))?; + } else if let Ok(start) = instance.get_typed_func::<(), ()>(&mut store, "_start") { + start.call(&mut store, ()).map_err(|e| WasmHostError::Instantiate(format!("_start: {e}")))?; } - // 缓存其它 exports let fn_on_context_create = instance .get_typed_func::<(u32, u32), ()>(&mut store, "proxy_on_context_create") .map_err(|e| WasmHostError::AbiViolation(format!("get proxy_on_context_create: {e}")))?; @@ -100,16 +115,20 @@ impl Vm { let fn_on_request_headers = instance .get_typed_func::<(u32, u32, u32), u32>(&mut store, "proxy_on_request_headers") .map_err(|e| WasmHostError::AbiViolation(format!("get proxy_on_request_headers: {e}")))?; + let fn_on_request_body = instance.get_typed_func::<(u32, u32, u32), u32>(&mut store, "proxy_on_request_body").ok(); + let fn_on_request_trailers = instance.get_typed_func::<(u32, u32), u32>(&mut store, "proxy_on_request_trailers").ok(); let fn_on_response_headers = instance .get_typed_func::<(u32, u32, u32), u32>(&mut store, "proxy_on_response_headers") .map_err(|e| WasmHostError::AbiViolation(format!("get proxy_on_response_headers: {e}")))?; let fn_on_response_body = instance.get_typed_func::<(u32, u32, u32), u32>(&mut store, "proxy_on_response_body").ok(); + let fn_on_response_trailers = instance.get_typed_func::<(u32, u32), u32>(&mut store, "proxy_on_response_trailers").ok(); let fn_on_http_call_response = instance .get_typed_func::<(u32, u32, u32, u32, u32), ()>(&mut store, "proxy_on_http_call_response") .map_err(|e| WasmHostError::AbiViolation(format!("get proxy_on_http_call_response: {e}")))?; let fn_on_log = instance.get_typed_func::(&mut store, "proxy_on_log").ok(); let fn_on_done = instance.get_typed_func::(&mut store, "proxy_on_done").ok(); let fn_on_delete = instance.get_typed_func::(&mut store, "proxy_on_delete").ok(); + let fn_on_tick = instance.get_typed_func::(&mut store, "proxy_on_tick").ok(); let root_id = store.data().root_context_id; let next_ctx_id = root_id + 1; @@ -126,12 +145,16 @@ impl Vm { fn_on_vm_start, fn_on_configure, fn_on_request_headers, + fn_on_request_body, + fn_on_request_trailers, fn_on_response_headers, fn_on_response_body, + fn_on_response_trailers, fn_on_http_call_response, fn_on_log, fn_on_done, fn_on_delete, + fn_on_tick, }; // 启动序：on_context_create(root, 0) → on_vm_start → on_configure @@ -140,8 +163,11 @@ impl Vm { if let Some(ref f) = vm.fn_on_vm_start { vm.store.data_mut().effective_context = root_id; let cfg_len = vm.store.data().configuration.len() as u32; - f.call(&mut vm.store, (root_id, cfg_len)) + let ok = f.call(&mut vm.store, (root_id, cfg_len)) .map_err(|e| WasmHostError::GuestTrap { hook: "on_vm_start", source: e })?; + if ok == 0 { + return Err(WasmHostError::Instantiate("guest on_vm_start returned 0 (=invalid VM configuration)".into())); + } } vm.store.data_mut().effective_context = root_id; let cfg_len = vm.store.data().configuration.len() as u32; @@ -164,36 +190,37 @@ impl Vm { Ok(()) } - /// 完整跑一遍：on_request_headers → 可能多次 dispatch → inner.call → on_response_* + /// 完整跑一遍：on_request_headers → 可能多次 dispatch → on_request_body → inner.call → on_response_* pub async fn process(&mut self, req: SgRequest, inner: spacegate_plugin::Inner) -> Result { + // 跨请求清理：上一次请求若提前 `send_local_response` 短路，可能留下未消费的 + // dispatch 结果和 pending token，不清掉会让本请求的 `drive_until_continue` + // 把陈旧响应误当成自己的（spec §proxy_http_call 不要求 host 持久化）。 + while self.dispatch_rx.try_recv().is_ok() {} + self.store.data_mut().pending_calls.clear(); + let http_ctx_id = self.next_ctx_id; self.next_ctx_id = self.next_ctx_id.wrapping_add(1); - // 把请求拆出来：pseudo headers + headers，存进 ctx 之前需要把数据全部 clone 出来 let (parts, body) = req.into_parts(); let method = parts.method.clone(); let uri = parts.uri.clone(); let version = parts.version; let path = uri.path_and_query().map(|p| p.to_string()).unwrap_or_else(|| "/".to_string()); let authority = uri.authority().map(|a| a.to_string()).unwrap_or_else(|| { - parts - .headers - .get(http::header::HOST) - .and_then(|h| h.to_str().ok()) - .unwrap_or("") - .to_string() + parts.headers.get(http::header::HOST).and_then(|h| h.to_str().ok()).unwrap_or("").to_string() }); let scheme = uri.scheme_str().unwrap_or("http").to_string(); - let mut headers = parts.headers.clone(); - // host 后续要根据 ctx 修改后写回，所以这份是 host 真实状态 + let headers = parts.headers.clone(); let pseudo = PseudoHeaders { method: method.as_str().to_string(), path: path.clone(), authority: authority.clone(), scheme, }; + let request_protocol = format!("{:?}", version); + + let want_request_body = self.fn_on_request_body.is_some(); - // 创建 http context let root_id = self.root_id; self.create_context(http_ctx_id, root_id)?; { @@ -204,72 +231,148 @@ impl Vm { ctx.request_pseudo = pseudo; ctx.request_headers = headers.clone(); ctx.continue_requested = false; + ctx.request_protocol = request_protocol; st.effective_context = http_ctx_id; } // 调 on_request_headers - let num_headers = (self.store.data().contexts[&http_ctx_id].request_headers.len() + 4) as u32; // +4 for pseudo + let num_headers = (self.store.data().contexts[&http_ctx_id].request_headers.len() + 4) as u32; + let end_of_stream_for_headers: u32 = if want_request_body { 0 } else { 1 }; let on_req_hdr = self.fn_on_request_headers.clone(); let action_raw = on_req_hdr - .call(&mut self.store, (http_ctx_id, num_headers, 1 /* end_of_stream=true 简化处理 */)) + .call(&mut self.store, (http_ctx_id, num_headers, end_of_stream_for_headers)) .map_err(|e| WasmHostError::GuestTrap { hook: "on_request_headers", source: e })?; let action = Action::from_u32(action_raw); debug!(target: "spacegate_plugin_wasm", http_ctx_id, ?action, "on_request_headers returned"); - // 处理 Pause：等异步回调 if action == Action::Pause { self.drive_until_continue(http_ctx_id).await?; } - // 主流程：driver_until_continue 内部仍是 async（等 mpsc），保留 await。 - // 检查 local_response if let Some(local) = self.store.data_mut().contexts.get_mut(&http_ctx_id).and_then(|c| c.local_response.take()) { - info!(target: "spacegate_plugin_wasm", http_ctx_id, status = local.status, "guest local response"); - // 调 on_log + on_done + on_delete - self.invoke_log_done_delete(http_ctx_id); + info!(target: "spacegate_plugin_wasm", http_ctx_id, status = local.status, "guest local response (after headers)"); + self.invoke_log_done_delete(http_ctx_id)?; return Ok(build_local_response(local)); } - // 把 ctx 内的 headers 写回 SgRequest - let new_headers = { - let ctx = self.store.data().contexts.get(&http_ctx_id); - ctx.map(|c| (c.request_headers.clone(), c.request_pseudo.clone())).unwrap_or_else(|| (HeaderMap::new(), PseudoHeaders::default())) + // ─── on_request_body：把请求 body 物化后喂给 guest（仅当 guest 导出该 hook）─── + let (new_req_for_inner, collected_body_after_hook) = if want_request_body { + // collect body + let collected = match body.collect().await { + Ok(c) => c.to_bytes(), + Err(_) => Bytes::new(), + }; + let body_size = collected.len() as u32; + { + let st = self.store.data_mut(); + if let Some(ctx) = st.contexts.get_mut(&http_ctx_id) { + ctx.request_body = Some(collected.clone()); + ctx.stage = ContextStage::RequestBody; + ctx.continue_requested = false; + ctx.request_size = collected.len() as u64; + st.effective_context = http_ctx_id; + } + } + let on_req_body = self.fn_on_request_body.clone().expect("guarded by want_request_body"); + let action_raw = on_req_body + .call(&mut self.store, (http_ctx_id, body_size, 1)) + .map_err(|e| WasmHostError::GuestTrap { hook: "on_request_body", source: e })?; + if Action::from_u32(action_raw) == Action::Pause { + self.drive_until_continue(http_ctx_id).await?; + } + if let Some(local) = self.store.data_mut().contexts.get_mut(&http_ctx_id).and_then(|c| c.local_response.take()) { + info!(target: "spacegate_plugin_wasm", http_ctx_id, status = local.status, "guest local response (after request body)"); + self.invoke_log_done_delete(http_ctx_id)?; + return Ok(build_local_response(local)); + } + let final_body = self + .store + .data() + .contexts + .get(&http_ctx_id) + .and_then(|c| c.request_body.clone()) + .unwrap_or(collected); + (None, Some(final_body)) + } else { + (Some(body), None) }; - headers = new_headers.0; - // 写回 host 真实状态：保留 method、重建 uri（path 可能被 guest 改） - let new_uri = rebuild_uri(&new_headers.1.scheme, &new_headers.1.authority, &new_headers.1.path).unwrap_or(uri); + + // ─── on_request_trailers：spacegate 当前不感知 trailers，给 guest 一个空 trailer 入参 ─── + if let Some(f) = self.fn_on_request_trailers.clone() { + self.store.data_mut().effective_context = http_ctx_id; + if let Some(ctx) = self.store.data_mut().contexts.get_mut(&http_ctx_id) { + ctx.stage = ContextStage::RequestTrailers; + ctx.continue_requested = false; + } + let action_raw = f + .call(&mut self.store, (http_ctx_id, 0)) + .map_err(|e| WasmHostError::GuestTrap { hook: "on_request_trailers", source: e })?; + if Action::from_u32(action_raw) == Action::Pause { + self.drive_until_continue(http_ctx_id).await?; + } + if let Some(local) = self.store.data_mut().contexts.get_mut(&http_ctx_id).and_then(|c| c.local_response.take()) { + info!(target: "spacegate_plugin_wasm", http_ctx_id, status = local.status, "guest local response (after request trailers)"); + self.invoke_log_done_delete(http_ctx_id)?; + return Ok(build_local_response(local)); + } + } + + // 把 ctx 内可能被 guest 改过的 method/path/headers 写回 SgRequest + let (new_headers, new_pseudo) = self + .store + .data() + .contexts + .get(&http_ctx_id) + .map(|c| (c.request_headers.clone(), c.request_pseudo.clone())) + .unwrap_or_else(|| (HeaderMap::new(), PseudoHeaders::default())); + let new_uri = rebuild_uri(&new_pseudo.scheme, &new_pseudo.authority, &new_pseudo.path).unwrap_or(uri); let mut new_parts = parts; - new_parts.method = new_headers.1.method.parse().unwrap_or(method); + new_parts.method = new_pseudo.method.parse().unwrap_or(method); new_parts.uri = new_uri; - new_parts.headers = headers; + new_parts.headers = new_headers; new_parts.version = version; - let new_req = SgRequest::from_parts(new_parts, body); + let new_body = match (new_req_for_inner, collected_body_after_hook) { + (Some(b), _) => b, + (None, Some(bytes)) => SgBody::full(bytes), + (None, None) => SgBody::empty(), + }; + let new_req = SgRequest::from_parts(new_parts, new_body); - // inner.call let resp = inner.call(new_req).await; - // on_response_headers + // ─── on_response_headers ─── let (resp_parts, resp_body) = resp.into_parts(); let status = resp_parts.status.as_u16(); + let status_message = resp_parts.status.canonical_reason().unwrap_or("").to_string(); let resp_headers = resp_parts.headers.clone(); { let st = self.store.data_mut(); if let Some(ctx) = st.contexts.get_mut(&http_ctx_id) { ctx.stage = ContextStage::ResponseHeaders; ctx.response_status = Some(status); + ctx.response_status_message = status_message; ctx.response_headers = resp_headers.clone(); ctx.continue_requested = false; st.effective_context = http_ctx_id; } } + let want_response_body = self.fn_on_response_body.is_some(); + let end_of_stream_for_resp_hdr: u32 = if want_response_body { 0 } else { 1 }; let on_resp_hdr = self.fn_on_response_headers.clone(); - let _ = on_resp_hdr - .call(&mut self.store, (http_ctx_id, (resp_headers.len() + 1) as u32, 1)) + let action_raw = on_resp_hdr + .call(&mut self.store, (http_ctx_id, (resp_headers.len() + 1) as u32, end_of_stream_for_resp_hdr)) .map_err(|e| WasmHostError::GuestTrap { hook: "on_response_headers", source: e })?; + if Action::from_u32(action_raw) == Action::Pause { + self.drive_until_continue(http_ctx_id).await?; + } + if let Some(local) = self.store.data_mut().contexts.get_mut(&http_ctx_id).and_then(|c| c.local_response.take()) { + info!(target: "spacegate_plugin_wasm", http_ctx_id, status = local.status, "guest local response (after response headers)"); + self.invoke_log_done_delete(http_ctx_id)?; + return Ok(build_local_response(local)); + } - // on_response_body：把 body dump 一次喂给 guest（首版非流式） - let on_resp_body = self.fn_on_response_body.clone(); - let (final_headers, final_body): (HeaderMap, SgBody) = if let Some(f) = on_resp_body { + // ─── on_response_body ─── + let (mut final_headers, final_body): (HeaderMap, SgBody) = if let Some(f) = self.fn_on_response_body.clone() { let collected = match resp_body.collect().await { Ok(c) => c.to_bytes(), Err(_) => Bytes::new(), @@ -280,13 +383,17 @@ impl Vm { if let Some(ctx) = st.contexts.get_mut(&http_ctx_id) { ctx.response_body = Some(collected.clone()); ctx.stage = ContextStage::ResponseBody; + ctx.continue_requested = false; + ctx.response_size = collected.len() as u64; st.effective_context = http_ctx_id; } } - let _ = f + let action_raw = f .call(&mut self.store, (http_ctx_id, body_size, 1)) .map_err(|e| WasmHostError::GuestTrap { hook: "on_response_body", source: e })?; - // 取回（guest 可能改过） + if Action::from_u32(action_raw) == Action::Pause { + self.drive_until_continue(http_ctx_id).await?; + } let updated_body = self.store.data().contexts.get(&http_ctx_id).and_then(|c| c.response_body.clone()).unwrap_or(collected); let updated_headers = self.store.data().contexts.get(&http_ctx_id).map(|c| c.response_headers.clone()).unwrap_or(resp_headers); (updated_headers, SgBody::full(updated_body)) @@ -294,8 +401,24 @@ impl Vm { (resp_headers, SgBody::new(resp_body)) }; - // on_log + on_done + on_delete - self.invoke_log_done_delete(http_ctx_id); + // ─── on_response_trailers ─── + if let Some(f) = self.fn_on_response_trailers.clone() { + self.store.data_mut().effective_context = http_ctx_id; + if let Some(ctx) = self.store.data_mut().contexts.get_mut(&http_ctx_id) { + ctx.stage = ContextStage::ResponseTrailers; + ctx.continue_requested = false; + } + let _ = f + .call(&mut self.store, (http_ctx_id, 0)) + .map_err(|e| WasmHostError::GuestTrap { hook: "on_response_trailers", source: e })?; + // guest 可能改了 response_headers → 同步回 final_headers + if let Some(ctx) = self.store.data().contexts.get(&http_ctx_id) { + final_headers = ctx.response_headers.clone(); + } + } + + // ─── on_log + on_done + on_delete ─── + self.invoke_log_done_delete(http_ctx_id)?; let mut new_resp_parts = resp_parts; new_resp_parts.headers = final_headers; @@ -303,10 +426,9 @@ impl Vm { } /// 在 guest 返回 Pause 之后，不停地 await dispatch_rx 来驱动状态机， - /// 直到 guest `continue_stream(Request)` 或写了 `local_response`。 + /// 直到 guest `continue_stream(HTTP_REQUEST/RESPONSE)` 或写了 `local_response`。 async fn drive_until_continue(&mut self, ctx_id: u32) -> Result<(), WasmHostError> { loop { - // 退出条件 { let st = self.store.data(); let Some(ctx) = st.contexts.get(&ctx_id) else { @@ -319,7 +441,6 @@ impl Vm { return Ok(()); } } - // 等下一个 dispatch 完成 let Some((token, result)) = self.dispatch_rx.recv().await else { return Err(WasmHostError::Dispatch("dispatch channel closed".to_string())); }; @@ -337,8 +458,8 @@ impl Vm { st.effective_context = source_ctx_id; if let Some(ctx) = st.contexts.get_mut(&source_ctx_id) { ctx.last_call_headers = result.headers.clone(); - // 单独存 status：HeaderMap 不接受 `:` key，pseudo_lookup 会读这里。 ctx.last_call_status = result.status; + ctx.last_call_status_message = result.status_message.clone(); ctx.last_call_body = result.body.clone(); ctx.continue_requested = false; } @@ -346,33 +467,71 @@ impl Vm { body_len = result.body.len() as u32; } debug!(target: "spacegate_plugin_wasm", token, source_ctx_id, status = result.status, body_len, "fire proxy_on_http_call_response"); - // 注意：proxy_on_http_call_response 通过 host fn 读取 last_call_*； - // 但 hai 通过 `get_http_call_response_header(":status")` 读 status： - // 我们 lookup_header 时对 HttpCallResponseHeaders 的 `:status` 做特判 let f = self.fn_on_http_call_response.clone(); f.call(&mut self.store, (source_ctx_id, token, header_count, body_len, 0)) .map_err(|e| WasmHostError::GuestTrap { hook: "on_http_call_response", source: e })?; } } - fn invoke_log_done_delete(&mut self, ctx_id: u32) { + fn invoke_log_done_delete(&mut self, ctx_id: u32) -> Result<(), WasmHostError> { self.store.data_mut().effective_context = ctx_id; + if let Some(ctx) = self.store.data_mut().contexts.get_mut(&ctx_id) { + ctx.stage = ContextStage::Log; + } if let Some(f) = self.fn_on_log.clone() { let _ = f.call(&mut self.store, ctx_id); } if let Some(f) = self.fn_on_done.clone() { - let _ = f.call(&mut self.store, ctx_id); + // spec §proxy_on_done：返回 false 表示 plugin 还要再调 `proxy_done`。 + // 当前 http context 在请求结束时即刻销毁，host 没有"再等一会"的空间： + // 标记 awaiting_done 让 `proxy_done` 能 Ok 一次，guest 若在 on_log 里立刻 done 则完美； + // 否则强制完成并 warn。 + if let Some(ctx) = self.store.data_mut().contexts.get_mut(&ctx_id) { + ctx.awaiting_done = true; + } + let v = f.call(&mut self.store, ctx_id).unwrap_or(1); + let done = v != 0 + || self + .store + .data() + .contexts + .get(&ctx_id) + .map(|c| c.done_marker) + .unwrap_or(true); + if !done { + warn!( + target: "spacegate_plugin_wasm", + ctx_id, + "proxy_on_done returned false but http context cannot defer; forcing delete" + ); + } } if let Some(f) = self.fn_on_delete.clone() { let _ = f.call(&mut self.store, ctx_id); } - // 清理 ctx self.store.data_mut().contexts.remove(&ctx_id); + Ok(()) } pub fn fail_strategy(&self) -> FailStrategy { self.fail_strategy } + + /// guest 当前请求的 `proxy_set_tick_period_milliseconds` 值；0 表示尚未配置 / 已停。 + pub fn tick_period_ms(&self) -> u32 { + self.store.data().tick_period_ms.unwrap_or(0) + } + + /// 在 root_context 上同步触发一次 `proxy_on_tick`。host 端后台任务调用本方法。 + /// + /// 失败要么是 guest trap（要么后台任务自停），要么是 guest 没导出 `proxy_on_tick`——后者直接 Ok。 + pub fn tick(&mut self) -> Result<(), WasmHostError> { + let Some(f) = self.fn_on_tick.clone() else { return Ok(()); }; + self.store.data_mut().effective_context = self.root_id; + f.call(&mut self.store, self.root_id) + .map_err(|e| WasmHostError::GuestTrap { hook: "on_tick", source: e })?; + Ok(()) + } } fn rebuild_uri(scheme: &str, authority: &str, path: &str) -> Option { @@ -399,77 +558,134 @@ fn build_local_response(local: crate::host_state::LocalResponse) -> SgResponse { resp } -/// 占位的 wasi_snapshot_preview1 hostcall：满足 hai_process_mix 的 _initialize 链接需求。 +/// spec §Unimplemented WASI functions + §Logging §Clocks §Randomness：完整的 wasi_snapshot_preview1 子集。 /// -/// - random_get / clock_time_get：用 host 端真实实现 -/// - environ_get / environ_sizes_get / fd_write / proc_exit：写到日志或返回 0 -fn register_wasi_stubs(linker: &mut Linker) -> Result<(), wasmtime::Error> { - // random_get(ptr, len) -> errno +/// - `random_get`：用 OS RNG（spec §Randomness）。 +/// - `clock_time_get`：spec §Clocks，REALTIME 用 SystemTime，MONOTONIC 用 Instant。 +/// - `environ_get` / `environ_sizes_get`：spec 明确不暴露 host env，全部 0。 +/// - `fd_write`：spec §Logging：fd=1→INFO，fd=2→ERROR；解析 iovec 提取 bytes。 +/// - `args_sizes_get` / `args_get`：spec §Unimplemented WASI，固定写 0。 +/// - `proc_exit`：spec §Unimplemented WASI，noop。 +pub fn register_wasi_stubs(linker: &mut Linker) -> Result<(), wasmtime::Error> { + use crate::abi::{wasi_errno, wasi_fd}; + linker.func_wrap( "wasi_snapshot_preview1", "random_get", |mut caller: wasmtime::Caller<'_, HostState>, ptr: i32, len: i32| -> i32 { - let mem = match crate::abi::MemoryHelper::from_caller(&mut caller) { + let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, - Err(_) => return 1, + Err(_) => return wasi_errno::FAULT, }; let mut buf = vec![0u8; len.max(0) as usize]; - // 简化：使用 SystemTime 的 nanos 做种子异或填充；不是密码学安全，对 hai 足够（hai 几乎不用 random） - let seed = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos() as u64) - .unwrap_or(0); - for (i, b) in buf.iter_mut().enumerate() { - *b = ((seed >> (i % 56)) as u8) ^ (i as u8); + if getrandom::getrandom(&mut buf).is_err() { + return wasi_errno::FAULT; + } + if mem.write_bytes(caller.as_context_mut(), ptr as u32, &buf).is_err() { + return wasi_errno::FAULT; } - let _ = mem.write_bytes(caller.as_context_mut(), ptr as u32, &buf); - 0 + wasi_errno::SUCCESS }, )?; - // clock_time_get(clock_id, precision, *result) -> errno linker.func_wrap( "wasi_snapshot_preview1", "clock_time_get", - |mut caller: wasmtime::Caller<'_, HostState>, _clock_id: i32, _prec: i64, return_ptr: i32| -> i32 { - let mem = match crate::abi::MemoryHelper::from_caller(&mut caller) { + |mut caller: wasmtime::Caller<'_, HostState>, clock_id: i32, _prec: i64, return_ptr: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, - Err(_) => return 1, + Err(_) => return wasi_errno::FAULT, + }; + let nanos: u64 = match clock_id { + 0 => std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos() as u64).unwrap_or(0), + 1 => { + static EPOCH: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); + let epoch = EPOCH.get_or_init(std::time::Instant::now); + epoch.elapsed().as_nanos() as u64 + } + _ => return wasi_errno::NOTSUP, }; - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos() as u64) - .unwrap_or(0); - let _ = mem.write_u64(caller.as_context_mut(), return_ptr as u32, nanos); - 0 + if mem.write_u64(caller.as_context_mut(), return_ptr as u32, nanos).is_err() { + return wasi_errno::FAULT; + } + wasi_errno::SUCCESS }, )?; - // environ_get(*environ, *environ_buf) -> errno - linker.func_wrap("wasi_snapshot_preview1", "environ_get", |_c: wasmtime::Caller<'_, HostState>, _a: i32, _b: i32| -> i32 { 0 })?; - // environ_sizes_get(*environc, *environ_buf_size) -> errno + linker.func_wrap("wasi_snapshot_preview1", "environ_get", |_c: wasmtime::Caller<'_, HostState>, _a: i32, _b: i32| -> i32 { wasi_errno::SUCCESS })?; linker.func_wrap( "wasi_snapshot_preview1", "environ_sizes_get", |mut caller: wasmtime::Caller<'_, HostState>, count_ptr: i32, buf_ptr: i32| -> i32 { - let mem = match crate::abi::MemoryHelper::from_caller(&mut caller) { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return wasi_errno::FAULT, + }; + if mem.write_u32(caller.as_context_mut(), count_ptr as u32, 0).is_err() { + return wasi_errno::FAULT; + } + if mem.write_u32(caller.as_context_mut(), buf_ptr as u32, 0).is_err() { + return wasi_errno::FAULT; + } + wasi_errno::SUCCESS + }, + )?; + linker.func_wrap("wasi_snapshot_preview1", "args_get", |_c: wasmtime::Caller<'_, HostState>, _a: i32, _b: i32| -> i32 { wasi_errno::SUCCESS })?; + linker.func_wrap( + "wasi_snapshot_preview1", + "args_sizes_get", + |mut caller: wasmtime::Caller<'_, HostState>, argc_ptr: i32, buf_size_ptr: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, - Err(_) => return 1, + Err(_) => return wasi_errno::FAULT, }; - let _ = mem.write_u32(caller.as_context_mut(), count_ptr as u32, 0); - let _ = mem.write_u32(caller.as_context_mut(), buf_ptr as u32, 0); - 0 + if mem.write_u32(caller.as_context_mut(), argc_ptr as u32, 0).is_err() { + return wasi_errno::FAULT; + } + if mem.write_u32(caller.as_context_mut(), buf_size_ptr as u32, 0).is_err() { + return wasi_errno::FAULT; + } + wasi_errno::SUCCESS }, )?; - // fd_write(fd, *iovs, iovs_len, *nwritten) -> errno —— hai 用 println 时会走这条，简单丢弃 linker.func_wrap( "wasi_snapshot_preview1", "fd_write", - |mut caller: wasmtime::Caller<'_, HostState>, _fd: i32, _iovs: i32, _iovs_len: i32, nwritten_ptr: i32| -> i32 { - let mem = match crate::abi::MemoryHelper::from_caller(&mut caller) { + |mut caller: wasmtime::Caller<'_, HostState>, fd: i32, iovs: i32, iovs_len: i32, nwritten_ptr: i32| -> i32 { + // spec §Logging：fd=1→INFO，fd=2→ERROR；其它 fd → BADF。 + if fd != wasi_fd::STDOUT && fd != wasi_fd::STDERR { + return wasi_errno::BADF; + } + let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, - Err(_) => return 1, + Err(_) => return wasi_errno::FAULT, }; - let _ = mem.write_u32(caller.as_context_mut(), nwritten_ptr as u32, 0); - 0 + // iovec[]：每项 (buf_ptr: u32, buf_len: u32)，共 iovs_len 项。 + let mut total: u32 = 0; + let mut bytes_out: Vec = Vec::new(); + for i in 0..(iovs_len as u32) { + let entry_ptr = (iovs as u32) + i * 8; + let Ok(buf_ptr) = mem.read_u32(caller.as_context(), entry_ptr) else { + return wasi_errno::FAULT; + }; + let Ok(buf_len) = mem.read_u32(caller.as_context(), entry_ptr + 4) else { + return wasi_errno::FAULT; + }; + let Ok(chunk) = mem.read_bytes(caller.as_context(), buf_ptr, buf_len) else { + return wasi_errno::FAULT; + }; + bytes_out.extend_from_slice(&chunk); + total = total.saturating_add(buf_len); + } + let msg = String::from_utf8_lossy(&bytes_out); + let msg_trimmed = msg.trim_end_matches('\n'); + if fd == wasi_fd::STDOUT { + tracing::info!(target: "spacegate_plugin_wasm::guest::stdout", "{msg_trimmed}"); + } else { + tracing::error!(target: "spacegate_plugin_wasm::guest::stderr", "{msg_trimmed}"); + } + if mem.write_u32(caller.as_context_mut(), nwritten_ptr as u32, total).is_err() { + return wasi_errno::FAULT; + } + wasi_errno::SUCCESS }, )?; linker.func_wrap("wasi_snapshot_preview1", "proc_exit", |_c: wasmtime::Caller<'_, HostState>, _code: i32| {})?; diff --git a/crates/plugin-wasm/tests/http_call.rs b/crates/plugin-wasm/tests/http_call.rs new file mode 100644 index 00000000..d3a21588 --- /dev/null +++ b/crates/plugin-wasm/tests/http_call.rs @@ -0,0 +1,179 @@ +//! 端到端验证 `proxy_http_call` → `proxy_on_http_call_response` 链路： +//! 模式来自 [`sdk_examples_guest`] 的 `auth_random`，guest 在 `on_request_headers` +//! 发起一次外呼，host 通过 reqwest 真正打到一个本地 mock HTTP server，server +//! 返回一段固定字节；guest 的 `on_http_call_response` 根据第一个字节决定 +//! `resume_http_request()` 放行 / `send_local_response(403)`。 +//! +//! 这条测试是 `proxy_http_call` 的唯一覆盖路径——host fn 注册、token 分配、 +//! reqwest spawn、UnboundedSender → drive_until_continue 状态机、effective_context +//! 切换、guest 通过 `get_http_call_response_body` 读 body —— 一次跑齐。 + +use std::convert::Infallible; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; +use http_body_util::{BodyExt, Full}; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{Request as HyperRequest, Response}; +use hyper_util::rt::TokioIo; +use spacegate_kernel::backend_service::ArcHyperService; +use spacegate_kernel::helper_layers::function::Inner; +use spacegate_kernel::{SgBody, SgRequest, SgResponse}; +use spacegate_plugin_wasm::config::WasmPluginShellConfig; +use spacegate_plugin_wasm::engine::shared_engine; +use spacegate_plugin_wasm::vm::Vm; +use tokio::net::TcpListener; +use wasmtime::Module; + +// ───────────────────────────────────────────────────────── +// 共用：定位 sdk_examples_guest.wasm（与 sdk_examples.rs 相同；故意复制以保持测试独立） +// ───────────────────────────────────────────────────────── + +fn guest_manifest_path() -> PathBuf { + let mut p = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + p.push("tests"); + p.push("sdk_examples_guest"); + p.push("Cargo.toml"); + p +} + +fn guest_wasm_path() -> PathBuf { + let manifest = guest_manifest_path(); + let out = std::process::Command::new(env!("CARGO")) + .args(["metadata", "--no-deps", "--format-version", "1", "--manifest-path"]) + .arg(&manifest) + .output() + .expect("cargo metadata: spawn"); + assert!(out.status.success(), "cargo metadata failed: {}", String::from_utf8_lossy(&out.stderr)); + let meta: serde_json::Value = serde_json::from_slice(&out.stdout).expect("parse cargo metadata json"); + let target_dir = meta["target_directory"].as_str().expect("target_directory missing"); + PathBuf::from(target_dir).join("wasm32-wasip1").join("release").join("sdk_examples_guest.wasm") +} + +fn ensure_guest_built() -> PathBuf { + let wasm = guest_wasm_path(); + if !wasm.exists() { + let status = std::process::Command::new(env!("CARGO")) + .args(["build", "--release", "--target", "wasm32-wasip1", "--manifest-path"]) + .arg(guest_manifest_path()) + .status() + .expect("cargo build: spawn"); + assert!(status.success(), "sdk_examples_guest build failed"); + assert!(wasm.exists(), "wasm still missing after build: {wasm:?}"); + } + wasm +} + +fn load_module() -> Arc { + let path = ensure_guest_built(); + let bytes = std::fs::read(&path).expect("read wasm"); + Arc::new(Module::new(shared_engine(), &bytes).expect("Module::new")) +} + +// ───────────────────────────────────────────────────────── +// mock HTTP server：返回单字节 body，用于驱动 auth_random 判断 +// ───────────────────────────────────────────────────────── + +async fn start_mock_server(body_byte: u8) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("local_addr"); + tokio::spawn(async move { + loop { + let (stream, _) = match listener.accept().await { + Ok(s) => s, + Err(_) => return, + }; + tokio::spawn(async move { + let svc = service_fn(move |_req: HyperRequest| async move { + let body = Bytes::from(vec![body_byte]); + let resp = Response::builder() + .status(200) + .body(Full::new(body)) + .expect("build resp"); + Ok::<_, Infallible>(resp) + }); + let _ = http1::Builder::new() + .serve_connection(TokioIo::new(stream), svc) + .await; + }); + } + }); + addr +} + +// ───────────────────────────────────────────────────────── +// mock inner.call：guest 放行后会下沉到这里，echo body 即可 +// ───────────────────────────────────────────────────────── + +fn echo_inner() -> Inner { + let svc = service_fn(|req: SgRequest| async move { + let (_, body) = req.into_parts(); + let bytes = body.collect().await.map(|c| c.to_bytes()).unwrap_or_default(); + let mut resp = SgResponse::new(SgBody::full(bytes)); + *resp.status_mut() = http::StatusCode::OK; + Ok::<_, Infallible>(resp) + }); + Inner::new(ArcHyperService::new(svc)) +} + +async fn full_body(resp: SgResponse) -> (SgResponse, Bytes) { + let (parts, body) = resp.into_parts(); + let bytes = body.collect().await.map(|c| c.to_bytes()).unwrap_or_default(); + (SgResponse::from_parts(parts, SgBody::full(bytes.clone())), bytes) +} + +async fn run(auth_byte: u8) -> (u16, Bytes) { + let addr = start_mock_server(auth_byte).await; + // 给 server 一个起跳的间隙；用 50ms 兜底（tokio 实际可即时 accept）。 + tokio::time::sleep(Duration::from_millis(20)).await; + + let module = load_module(); + let cfg = Arc::new(WasmPluginShellConfig { + url: "file://sdk_examples_guest".into(), + plugin_config: serde_json::json!({ + "mode": "auth_random", + "auth_cluster": "auth", + "auth_threshold": 128 + }), + clusters: [("auth".to_string(), format!("http://{addr}"))].into_iter().collect(), + ..Default::default() + }); + let mut vm = Vm::new(&module, cfg).expect("Vm::new"); + + let req = HyperRequest::builder() + .method("POST") + .uri("http://example.test/") + .header("host", "example.test") + .body(SgBody::full(Bytes::from_static(b"protected payload"))) + .expect("build req"); + + let resp = vm.process(req, echo_inner()).await.expect("process"); + let (resp, body) = full_body(resp).await; + (resp.status().as_u16(), body) +} + +// ───────────────────────────────────────────────────────── +// auth byte < threshold → 放行；echo 回原 body +// ───────────────────────────────────────────────────────── + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn auth_random_allow() { + let (status, body) = run(50).await; + assert_eq!(status, 200, "expected allow → echo"); + assert_eq!(body, Bytes::from_static(b"protected payload")); +} + +// ───────────────────────────────────────────────────────── +// auth byte >= threshold → guest 短路 403 +// ───────────────────────────────────────────────────────── + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn auth_random_deny() { + let (status, body) = run(200).await; + assert_eq!(status, 403, "expected deny → 403"); + assert_eq!(body, Bytes::from_static(b"forbidden")); +} diff --git a/crates/plugin-wasm/tests/on_tick.rs b/crates/plugin-wasm/tests/on_tick.rs new file mode 100644 index 00000000..0597c340 --- /dev/null +++ b/crates/plugin-wasm/tests/on_tick.rs @@ -0,0 +1,104 @@ +//! 端到端验证 `proxy_set_tick_period_milliseconds` + `proxy_on_tick` ⇄ host VmPool +//! 后台 tick 任务的协同： +//! +//! 1. `WasmPluginShell::create` 后，shell 内部起一条 50ms 颗粒度的 tick 循环； +//! 2. guest 在 `on_vm_start` 把 tick 周期设为 50ms，`on_tick` 把 shared_data 计数原子 +1； +//! 3. 测试 sleep 几个 tick 周期后从 host 侧直接读 shared_data，断言至少 N 次 tick； +//! 4. `drop(shell)` 后再 sleep，确认计数不再继续增长（tick 任务随 shell 析构）。 + +use std::path::PathBuf; +use std::time::Duration; + +use spacegate_plugin::{Plugin, PluginConfig}; +use spacegate_plugin_wasm::shared::{shared_data_get, shared_data_set}; +use spacegate_plugin_wasm::WasmPluginShell; +use spacegate_model::{PluginInstanceId, PluginInstanceName}; + +// ───────────────────────────────────────────────────────── +// guest .wasm 定位/构建 +// ───────────────────────────────────────────────────────── + +fn guest_manifest_path() -> PathBuf { + let mut p = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + p.push("tests"); + p.push("on_tick_guest"); + p.push("Cargo.toml"); + p +} + +fn guest_wasm_path() -> PathBuf { + let manifest = guest_manifest_path(); + let out = std::process::Command::new(env!("CARGO")) + .args(["metadata", "--no-deps", "--format-version", "1", "--manifest-path"]) + .arg(&manifest) + .output() + .expect("cargo metadata: spawn"); + assert!(out.status.success(), "cargo metadata failed: {}", String::from_utf8_lossy(&out.stderr)); + let meta: serde_json::Value = serde_json::from_slice(&out.stdout).expect("parse cargo metadata json"); + let target_dir = meta["target_directory"].as_str().expect("target_directory missing"); + PathBuf::from(target_dir).join("wasm32-wasip1").join("release").join("on_tick_guest.wasm") +} + +fn ensure_guest_built() -> PathBuf { + let wasm = guest_wasm_path(); + if !wasm.exists() { + let status = std::process::Command::new(env!("CARGO")) + .args(["build", "--release", "--target", "wasm32-wasip1", "--manifest-path"]) + .arg(guest_manifest_path()) + .status() + .expect("cargo build: spawn"); + assert!(status.success(), "on_tick_guest build failed"); + assert!(wasm.exists(), "wasm still missing after build: {wasm:?}"); + } + wasm +} + +fn read_counter() -> u64 { + let (raw, _cas) = shared_data_get(b"on_tick.count").expect("counter present"); + std::str::from_utf8(&raw).ok().and_then(|s| s.parse::().ok()).unwrap_or(0) +} + +// ───────────────────────────────────────────────────────── +// 主测试 +// ───────────────────────────────────────────────────────── + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn proxy_on_tick_drives_background_ticks() { + // 把 shared_data 计数器清零（cas=0 = 无 CAS 期望）。 + let _ = shared_data_set(b"on_tick.count", b"0", 0); + + let wasm = ensure_guest_built(); + let plugin_config = PluginConfig { + id: PluginInstanceId { + code: "wasm".into(), + name: PluginInstanceName::named("on-tick-test"), + }, + spec: serde_json::json!({ + "url": format!("file://{}", wasm.display()), + "plugin_name": "on-tick-plugin", + "plugin_root_id": "on-tick-root", + "plugin_vm_id": "default", + }), + }; + let shell = WasmPluginShell::create(plugin_config).expect("Plugin::create"); + + // shell 内部已经 spawn 了 50ms 颗粒度的 tick 任务； + // 期间 guest `on_vm_start` 把 period 设成 50ms。 + // 等 300ms 至少 4 次 tick（保留调度抖动余量）。 + tokio::time::sleep(Duration::from_millis(300)).await; + + let count = read_counter(); + assert!(count >= 4, "expected >= 4 ticks in 300ms, got {count}"); + tracing::info!("got {count} ticks"); + + // 取一次 snapshot，drop 之后再 sleep 同等时间，断言不再继续增长（允许 1 次余量： + // task abort 与正在执行中的同步 tick() 之间可能交叠一次）。 + let snapshot = count; + drop(shell); + tokio::time::sleep(Duration::from_millis(200)).await; + let after = read_counter(); + assert!( + after.saturating_sub(snapshot) <= 1, + "tick task should stop after shell drop: snapshot={snapshot}, after={after}", + ); +} diff --git a/crates/plugin-wasm/tests/on_tick_guest/.cargo/config.toml b/crates/plugin-wasm/tests/on_tick_guest/.cargo/config.toml new file mode 100644 index 00000000..6b509f5b --- /dev/null +++ b/crates/plugin-wasm/tests/on_tick_guest/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +target = "wasm32-wasip1" diff --git a/crates/plugin-wasm/tests/on_tick_guest/Cargo.toml b/crates/plugin-wasm/tests/on_tick_guest/Cargo.toml new file mode 100644 index 00000000..6fb9b844 --- /dev/null +++ b/crates/plugin-wasm/tests/on_tick_guest/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "on_tick_guest" +version = "0.0.0" +edition = "2021" +publish = false +description = "Tiny proxy-wasm guest exercising proxy_set_tick_period_milliseconds + proxy_on_tick; persists tick count into shared_data for host-side assertion." + +[workspace] + +[lib] +crate-type = ["cdylib"] + +[dependencies] +proxy-wasm = "0.2" + +[profile.release] +codegen-units = 1 +opt-level = "z" +lto = "fat" +strip = true +panic = "abort" diff --git a/crates/plugin-wasm/tests/on_tick_guest/src/lib.rs b/crates/plugin-wasm/tests/on_tick_guest/src/lib.rs new file mode 100644 index 00000000..f859de48 --- /dev/null +++ b/crates/plugin-wasm/tests/on_tick_guest/src/lib.rs @@ -0,0 +1,46 @@ +//! 验证 host 端 VmPool + 后台 tick 任务能正确驱动 `proxy_on_tick`。 +//! +//! - `on_vm_start`：设置 50ms 的 tick 周期； +//! - `on_tick`：把全局 tick 计数器（shared_data，key="on_tick.count"）原子地 +1； +//! +//! 测试侧通过 `spacegate_plugin_wasm::shared::shared_data_get` 直接读 shared_data， +//! 等若干 tick 之后断言计数大于 0。 +//! +//! `set_shared_data` 用 cas-loop 保证多 VM / 后台并发也不会丢更新（虽然现在只有一条 tick 任务）。 + +use std::time::Duration; + +use proxy_wasm::hostcalls; +use proxy_wasm::traits::*; +use proxy_wasm::types::*; + +const KEY: &str = "on_tick.count"; + +proxy_wasm::main! {{ + proxy_wasm::set_log_level(LogLevel::Info); + proxy_wasm::set_root_context(|_| -> Box { Box::new(TickRoot) }); +}} + +struct TickRoot; +impl Context for TickRoot {} +impl RootContext for TickRoot { + fn on_vm_start(&mut self, _: usize) -> bool { + // 50ms 周期：host 端默认 50ms 颗粒度的轮询正好能驱动。 + let _ = hostcalls::set_tick_period(Duration::from_millis(50)); + true + } + + fn on_tick(&mut self) { + // cas 循环：读 → +1 → 写；写失败 (CasMismatch) 重读重试。 + for _ in 0..8 { + let (cur, cas) = hostcalls::get_shared_data(KEY).unwrap_or((None, None)); + let next = cur.as_deref().and_then(|b| std::str::from_utf8(b).ok()).and_then(|s| s.parse::().ok()).unwrap_or(0) + 1; + let buf = next.to_string(); + match hostcalls::set_shared_data(KEY, Some(buf.as_bytes()), cas) { + Ok(()) => return, + Err(Status::CasMismatch) => continue, + Err(_) => return, + } + } + } +} diff --git a/crates/plugin-wasm/tests/sdk_examples.rs b/crates/plugin-wasm/tests/sdk_examples.rs new file mode 100644 index 00000000..4c21be5c --- /dev/null +++ b/crates/plugin-wasm/tests/sdk_examples.rs @@ -0,0 +1,265 @@ +//! 用 `proxy-wasm-rust-sdk` 仓库 `examples/` 的 4 个范例对应行为构造 +//! [`sdk_examples_guest`] 单 wasm 多模式，逐个跑完整 [`Vm::process`] 链路。 +//! +//! 这个测试是真正的端到端：plugin configuration → on_configure → 请求进入插件 → +//! on_request_headers / on_request_body → inner.call（我们 mock 出来的 hyper 服务）→ +//! on_response_headers → 最终响应 → on_log。它直接证明： +//! +//! - SDK 标准范例的 host fn 调用面我们 host 全部正确实现； +//! - body 改写、本地响应短路、required header 拦截 这些跨阶段的协同没问题。 +//! +//! 本文件 **不** 覆盖 `auth_random`（需要 mock HTTP server 走 reqwest），那部分在 +//! [`tests/http_call.rs`] 单独测。 + +use std::convert::Infallible; +use std::path::PathBuf; +use std::sync::Arc; + +use bytes::Bytes; +use http_body_util::BodyExt; +use hyper::service::service_fn; +use spacegate_kernel::backend_service::ArcHyperService; +use spacegate_kernel::helper_layers::function::Inner; +use hyper::Request as HyperRequest; +use spacegate_kernel::{SgBody, SgRequest, SgResponse}; +use spacegate_plugin_wasm::config::WasmPluginShellConfig; +use spacegate_plugin_wasm::engine::shared_engine; +use spacegate_plugin_wasm::vm::Vm; +use wasmtime::Module; + +// ───────────────────────────────────────────────────────── +// 公共：定位/构建 sdk_examples_guest.wasm +// ───────────────────────────────────────────────────────── + +fn guest_manifest_path() -> PathBuf { + let mut p = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + p.push("tests"); + p.push("sdk_examples_guest"); + p.push("Cargo.toml"); + p +} + +fn guest_wasm_path() -> PathBuf { + let manifest = guest_manifest_path(); + let out = std::process::Command::new(env!("CARGO")) + .args(["metadata", "--no-deps", "--format-version", "1", "--manifest-path"]) + .arg(&manifest) + .output() + .expect("cargo metadata: spawn"); + assert!(out.status.success(), "cargo metadata failed: {}", String::from_utf8_lossy(&out.stderr)); + let meta: serde_json::Value = serde_json::from_slice(&out.stdout).expect("parse cargo metadata json"); + let target_dir = meta["target_directory"].as_str().expect("target_directory missing"); + PathBuf::from(target_dir).join("wasm32-wasip1").join("release").join("sdk_examples_guest.wasm") +} + +fn ensure_guest_built() -> PathBuf { + let wasm = guest_wasm_path(); + if !wasm.exists() { + eprintln!("[sdk_examples] building sdk_examples_guest …"); + let status = std::process::Command::new(env!("CARGO")) + .args(["build", "--release", "--target", "wasm32-wasip1", "--manifest-path"]) + .arg(guest_manifest_path()) + .status() + .expect("cargo build: spawn"); + assert!(status.success(), "sdk_examples_guest build failed"); + assert!(wasm.exists(), "wasm still missing after build: {wasm:?}"); + } + wasm +} + +fn load_module() -> Arc { + let path = ensure_guest_built(); + let bytes = std::fs::read(&path).expect("read wasm"); + let module = Module::new(shared_engine(), &bytes).expect("Module::new"); + Arc::new(module) +} + +// ───────────────────────────────────────────────────────── +// mock `Inner`：把请求 body 原样回显，并复制 `x-echo-*` 头 +// ───────────────────────────────────────────────────────── + +#[derive(Clone, Default)] +struct CaptureState { + /// inner.call 真正收到的请求体（guest 修改后下沉的内容） + inbound_body: Arc>>, + /// inner.call 实际是否被调到（验证 send_local_response 的短路） + invoked: Arc, +} + +fn make_inner(state: CaptureState) -> Inner { + let svc = service_fn(move |req: SgRequest| { + let state = state.clone(); + async move { + state.invoked.store(true, std::sync::atomic::Ordering::SeqCst); + let (parts, body) = req.into_parts(); + let bytes = body.collect().await.map(|c| c.to_bytes()).unwrap_or_default(); + *state.inbound_body.lock().await = Some(bytes.clone()); + let mut resp = SgResponse::new(SgBody::full(bytes)); + for (k, v) in parts.headers.iter() { + if k.as_str().starts_with("x-echo-") { + resp.headers_mut().insert(k, v.clone()); + } + } + Ok::<_, Infallible>(resp) + } + }); + Inner::new(ArcHyperService::new(svc)) +} + +fn make_cfg(spec: serde_json::Value) -> Arc { + Arc::new(WasmPluginShellConfig { + url: "file://sdk_examples_guest".into(), + plugin_config: spec, + plugin_name: "sdk-examples-test".into(), + plugin_root_id: "sdk-examples-root".into(), + plugin_vm_id: "default".into(), + ..Default::default() + }) +} + +async fn full_body(resp: SgResponse) -> (SgResponse, Bytes) { + let (parts, body) = resp.into_parts(); + let bytes = body.collect().await.map(|c| c.to_bytes()).unwrap_or_default(); + (SgResponse::from_parts(parts, SgBody::full(bytes.clone())), bytes) +} + +// ───────────────────────────────────────────────────────── +// 1. http_headers：/hello → 本地 200 + Hello/Powered-By；其余 → 走 inner，加 x-sdk-headers 响应头 +// ───────────────────────────────────────────────────────── + +#[tokio::test] +async fn sdk_example_http_headers_hello() { + let module = load_module(); + let cfg = make_cfg(serde_json::json!({"mode": "headers"})); + let mut vm = Vm::new(&module, cfg).expect("Vm::new"); + + let req = HyperRequest::builder() + .method("GET") + .uri("http://example.test/hello") + .header("host", "example.test") + .body(SgBody::empty()) + .expect("build req"); + let captured = CaptureState::default(); + let inner = make_inner(captured.clone()); + let resp = vm.process(req, inner).await.expect("process"); + let (resp, body) = full_body(resp).await; + + assert_eq!(resp.status(), 200); + assert_eq!(body, Bytes::from_static(b"Hello, World!\n")); + assert_eq!(resp.headers().get("hello").and_then(|v| v.to_str().ok()), Some("world")); + assert_eq!( + resp.headers().get("powered-by").and_then(|v| v.to_str().ok()), + Some("proxy-wasm") + ); + assert!( + !captured.invoked.load(std::sync::atomic::Ordering::SeqCst), + "inner.call must NOT be invoked for local response" + ); +} + +#[tokio::test] +async fn sdk_example_http_headers_passthrough() { + let module = load_module(); + let cfg = make_cfg(serde_json::json!({"mode": "headers"})); + let mut vm = Vm::new(&module, cfg).expect("Vm::new"); + + let req = HyperRequest::builder() + .method("GET") + .uri("http://example.test/world") + .header("host", "example.test") + .header("x-echo-foo", "bar") + .body(SgBody::empty()) + .expect("build req"); + let captured = CaptureState::default(); + let resp = vm.process(req, make_inner(captured.clone())).await.expect("process"); + let (resp, _body) = full_body(resp).await; + + assert_eq!(resp.status(), 200); + assert!(captured.invoked.load(std::sync::atomic::Ordering::SeqCst)); + assert_eq!( + resp.headers().get("x-sdk-headers").and_then(|v| v.to_str().ok()), + Some("seen"), + "on_response_headers should inject x-sdk-headers" + ); + // echo header 应该原路回来 + assert_eq!( + resp.headers().get("x-echo-foo").and_then(|v| v.to_str().ok()), + Some("bar") + ); +} + +// ───────────────────────────────────────────────────────── +// 2. http_body：on_request_body 把 body 反转后下沉给 inner.call +// ───────────────────────────────────────────────────────── + +#[tokio::test] +async fn sdk_example_http_body_reverses_request_body() { + let module = load_module(); + let cfg = make_cfg(serde_json::json!({"mode": "body"})); + let mut vm = Vm::new(&module, cfg).expect("Vm::new"); + + let req = HyperRequest::builder() + .method("POST") + .uri("http://example.test/reverse") + .header("host", "example.test") + .body(SgBody::full(Bytes::from_static(b"abc-123"))) + .expect("build req"); + let captured = CaptureState::default(); + let resp = vm.process(req, make_inner(captured.clone())).await.expect("process"); + let (resp, body) = full_body(resp).await; + + assert_eq!(resp.status(), 200); + // Inner 收到的应是反转后的字节，echo 回来后响应体也是它。 + assert_eq!( + captured.inbound_body.lock().await.clone().expect("body captured"), + Bytes::from_static(b"321-cba") + ); + assert_eq!(body, Bytes::from_static(b"321-cba")); +} + +// ───────────────────────────────────────────────────────── +// 3. http_config：缺失 x-token → 本地 403；带上 → 放行 +// ───────────────────────────────────────────────────────── + +#[tokio::test] +async fn sdk_example_http_config_missing_header_rejected() { + let module = load_module(); + let cfg = make_cfg(serde_json::json!({"mode": "config", "required_header": "x-token"})); + let mut vm = Vm::new(&module, cfg).expect("Vm::new"); + + let req = HyperRequest::builder() + .method("GET") + .uri("http://example.test/") + .header("host", "example.test") + .body(SgBody::empty()) + .expect("build req"); + let captured = CaptureState::default(); + let resp = vm.process(req, make_inner(captured.clone())).await.expect("process"); + let (resp, body) = full_body(resp).await; + + assert_eq!(resp.status(), 403); + assert_eq!(body, Bytes::from_static(b"missing required header")); + assert!(!captured.invoked.load(std::sync::atomic::Ordering::SeqCst)); +} + +#[tokio::test] +async fn sdk_example_http_config_present_header_passthrough() { + let module = load_module(); + let cfg = make_cfg(serde_json::json!({"mode": "config", "required_header": "x-token"})); + let mut vm = Vm::new(&module, cfg).expect("Vm::new"); + + let req = HyperRequest::builder() + .method("GET") + .uri("http://example.test/") + .header("host", "example.test") + .header("x-token", "abc") + .body(SgBody::full(Bytes::from_static(b"hello"))) + .expect("build req"); + let captured = CaptureState::default(); + let resp = vm.process(req, make_inner(captured.clone())).await.expect("process"); + let (resp, body) = full_body(resp).await; + + assert_eq!(resp.status(), 200); + assert_eq!(body, Bytes::from_static(b"hello")); + assert!(captured.invoked.load(std::sync::atomic::Ordering::SeqCst)); +} diff --git a/crates/plugin-wasm/tests/sdk_examples_guest/.cargo/config.toml b/crates/plugin-wasm/tests/sdk_examples_guest/.cargo/config.toml new file mode 100644 index 00000000..6b509f5b --- /dev/null +++ b/crates/plugin-wasm/tests/sdk_examples_guest/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +target = "wasm32-wasip1" diff --git a/crates/plugin-wasm/tests/sdk_examples_guest/Cargo.toml b/crates/plugin-wasm/tests/sdk_examples_guest/Cargo.toml new file mode 100644 index 00000000..727907a2 --- /dev/null +++ b/crates/plugin-wasm/tests/sdk_examples_guest/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "sdk_examples_guest" +version = "0.0.0" +edition = "2021" +publish = false +description = "Mirrors proxy-wasm-rust-sdk examples (http_headers / http_body / http_config / http_auth_random) inside one guest, selectable via plugin configuration." + +# 独立 workspace：本 crate 目标 `wasm32-wasip1`，不参与外层 host workspace。 +[workspace] + +[lib] +crate-type = ["cdylib"] + +[dependencies] +proxy-wasm = "0.2" +log = "0.4" + +[profile.release] +codegen-units = 1 +opt-level = "z" +lto = "fat" +strip = true +panic = "abort" diff --git a/crates/plugin-wasm/tests/sdk_examples_guest/src/lib.rs b/crates/plugin-wasm/tests/sdk_examples_guest/src/lib.rs new file mode 100644 index 00000000..35223020 --- /dev/null +++ b/crates/plugin-wasm/tests/sdk_examples_guest/src/lib.rs @@ -0,0 +1,207 @@ +//! 把 `proxy-wasm/proxy-wasm-rust-sdk` 仓库 `examples/` 下 4 个范例汇总进同一个 guest， +//! 由 `on_configure` 读到的 plugin configuration 选择运行模式： +//! +//! - `mode: headers` ←→ examples/http_headers （读 req/resp 头 + `/hello` 本地响应） +//! - `mode: body` ←→ examples/http_body （on_request_body 反转字节后落到 inner.call） +//! - `mode: config` ←→ examples/http_config （要求请求带某个 header，缺失则 403） +//! - `mode: auth_random` ←→ examples/http_auth_random （`proxy_http_call` 到 "auth" cluster 决定放行） +//! +//! 之所以做成单 wasm + 模式切换，是为了集成测试只需构建一次 wasm 即可覆盖所有 SDK 范例。 +//! +//! configuration 直接吃明文 YAML：第一行 `mode: `，第二行（可选）模式相关参数。 +//! 这样不引入 serde_yaml/serde_json 依赖，wasm 体积更小、装配速度更快。 + +use log::{info, warn}; +use proxy_wasm::traits::*; +use proxy_wasm::types::*; + +proxy_wasm::main! {{ + proxy_wasm::set_log_level(LogLevel::Trace); + proxy_wasm::set_root_context(|_| -> Box { Box::new(SdkRoot::default()) }); +}} + +#[derive(Default, Clone)] +struct SdkConfig { + mode: Mode, + /// `config` 模式：缺失即 403 的请求头名字（默认 `x-token`）。 + required_header: String, + /// `auth_random` 模式：放行阈值。host 给出的 random byte < threshold 则视为允许。 + auth_threshold: u8, + /// `auth_random` 模式：用于 dispatch_http_call 的 cluster 名。 + auth_cluster: String, +} + +#[derive(Default, Clone, Copy, PartialEq, Eq)] +enum Mode { + #[default] + Noop, + Headers, + Body, + Config, + AuthRandom, +} + +impl Mode { + fn parse(s: &str) -> Self { + match s.trim() { + "headers" => Mode::Headers, + "body" => Mode::Body, + "config" => Mode::Config, + "auth_random" => Mode::AuthRandom, + _ => Mode::Noop, + } + } +} + +#[derive(Default)] +struct SdkRoot { + cfg: SdkConfig, +} + +impl Context for SdkRoot {} + +impl RootContext for SdkRoot { + fn on_vm_start(&mut self, _: usize) -> bool { true } + + fn on_configure(&mut self, _: usize) -> bool { + let raw = self.get_plugin_configuration().unwrap_or_default(); + let text = String::from_utf8_lossy(&raw); + let mut cfg = SdkConfig::default(); + cfg.required_header = "x-token".into(); + cfg.auth_threshold = 128; + cfg.auth_cluster = "auth".into(); + for line in text.lines() { + let Some((k, v)) = line.split_once(':') else { continue }; + // 去掉 YAML 单/双引号 + let v = v.trim().trim_matches(['"', '\''].as_ref()); + match k.trim() { + "mode" => cfg.mode = Mode::parse(v), + "required_header" => cfg.required_header = v.to_string(), + "auth_threshold" => cfg.auth_threshold = v.parse().unwrap_or(128), + "auth_cluster" => cfg.auth_cluster = v.to_string(), + _ => {} + } + } + info!("sdk_examples_guest configured: mode_set={}", cfg.mode != Mode::Noop); + self.cfg = cfg; + true + } + + fn create_http_context(&self, _context_id: u32) -> Option> { + Some(Box::new(SdkHttp { + cfg: self.cfg.clone(), + pending_token: None, + })) + } + + fn get_type(&self) -> Option { + Some(ContextType::HttpContext) + } +} + +struct SdkHttp { + cfg: SdkConfig, + pending_token: Option, +} + +impl Context for SdkHttp { + fn on_http_call_response(&mut self, token_id: u32, _num_headers: usize, body_size: usize, _num_trailers: usize) { + // 只 auth_random 模式会到这里；其它模式根本没发 dispatch。 + if Some(token_id) != self.pending_token { return; } + self.pending_token = None; + let body = self + .get_http_call_response_body(0, body_size) + .unwrap_or_default(); + let allow = body.first().map(|b| *b < self.cfg.auth_threshold).unwrap_or(false); + if allow { + // 放行：把 Pause 恢复 → 让 host 继续 inner.call。 + self.resume_http_request(); + } else { + self.send_http_response(403, vec![("x-rejected-by", "auth_random")], Some(b"forbidden")); + } + } +} + +impl HttpContext for SdkHttp { + fn on_http_request_headers(&mut self, _: usize, _: bool) -> Action { + match self.cfg.mode { + Mode::Headers => { + for (name, value) in &self.get_http_request_headers() { + info!("-> {name}: {value}"); + } + match self.get_http_request_header(":path") { + Some(p) if p == "/hello" => { + self.send_http_response( + 200, + vec![("hello", "world"), ("powered-by", "proxy-wasm")], + Some(b"Hello, World!\n"), + ); + Action::Pause + } + _ => Action::Continue, + } + } + Mode::Body => Action::Continue, + Mode::Config => { + if self.get_http_request_header(&self.cfg.required_header).is_some() { + Action::Continue + } else { + self.send_http_response( + 403, + vec![("x-rejected-by", "http_config")], + Some(b"missing required header"), + ); + Action::Pause + } + } + Mode::AuthRandom => { + match self.dispatch_http_call( + &self.cfg.auth_cluster.clone(), + vec![(":method", "GET"), (":path", "/random"), (":authority", "auth")], + None, + vec![], + std::time::Duration::from_millis(500), + ) { + Ok(token) => { + self.pending_token = Some(token); + Action::Pause + } + Err(s) => { + warn!("dispatch_http_call failed: status={s:?}"); + self.send_http_response(502, vec![("x-rejected-by", "dispatch_failed")], Some(b"upstream auth unreachable")); + Action::Pause + } + } + } + Mode::Noop => Action::Continue, + } + } + + fn on_http_request_body(&mut self, body_size: usize, end_of_stream: bool) -> Action { + if !matches!(self.cfg.mode, Mode::Body) || !end_of_stream { + return Action::Continue; + } + if body_size == 0 { return Action::Continue; } + let body = self.get_http_request_body(0, body_size).unwrap_or_default(); + let mut rev = body.clone(); + rev.reverse(); + // spec §Buffers：start=0, size=原长度 → 替换；len(value)=新长度。 + let _ = self.set_http_request_body(0, body.len(), &rev); + Action::Continue + } + + fn on_http_response_headers(&mut self, _: usize, _: bool) -> Action { + if matches!(self.cfg.mode, Mode::Headers) { + for (name, value) in &self.get_http_response_headers() { + info!("<- {name}: {value}"); + } + // 给响应额外塞一条头：SDK 那个 example 没塞，我们这里塞便于测试断言。 + let _ = self.add_http_response_header("x-sdk-headers", "seen"); + } + Action::Continue + } + + fn on_log(&mut self) { + info!("sdk_examples_guest: ctx done."); + } +} diff --git a/crates/plugin-wasm/tests/spec_compliance.rs b/crates/plugin-wasm/tests/spec_compliance.rs new file mode 100644 index 00000000..13f6fabe --- /dev/null +++ b/crates/plugin-wasm/tests/spec_compliance.rs @@ -0,0 +1,236 @@ +//! End-to-end spec compliance test：用 `proxy-wasm-rust-sdk` 编出来的真实 guest 插件 +//! ([`crates/plugin-wasm/tests/spec_test_guest`]) 跑一遍我们 host 注册的所有 hostcall， +//! 覆盖 proxy-wasm v0.2.1 spec 关键面： +//! +//! - Shared K/V（带 CAS） +//! - Shared queues（含 register/resolve/enqueue/dequeue 全链） +//! - Metrics（counter / gauge / record / increment） +//! - Properties（user + well-known `plugin_name`） +//! - Logging / Clocks +//! - Buffer（PluginConfiguration） +//! - HTTP header map（含 `:method` 伪头） +//! - Stream control（continue / close） +//! - effective_context / done +//! - gRPC / foreign_function 的 spec 合规返回值 +//! - send_local_response 短路写入 +//! - set_tick_period 接收 +//! +//! 运行：先 `cd crates/plugin-wasm/tests/spec_test_guest && cargo build --release` +//! 生成 wasm，再 `cargo test -p spacegate-plugin-wasm --test spec_compliance`。 +//! 测试入口会通过 cargo metadata 自动定位 wasm 路径并按需在缺失时调用 cargo 触发构建。 + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; + +use bytes::Bytes; +use http::HeaderMap; +use spacegate_plugin_wasm::config::WasmPluginShellConfig; +use spacegate_plugin_wasm::engine::shared_engine; +use spacegate_plugin_wasm::host_fn::register_all; +use spacegate_plugin_wasm::host_state::{ContextStage, HostState, HttpCallResult, PseudoHeaders, RequestContext}; +use spacegate_plugin_wasm::vm::register_wasi_stubs; +use wasmtime::{Instance, Linker, Module, Store, TypedFunc}; + +const HTTP_CONTEXT_ID: u32 = 2; + +// ───────────────────────────────────────────────────────── +// 定位 guest wasm；缺失则触发一次 `cargo build --release` +// ───────────────────────────────────────────────────────── + +fn guest_manifest_path() -> PathBuf { + let mut p = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + p.push("tests"); + p.push("spec_test_guest"); + p.push("Cargo.toml"); + p +} + +/// 用 `cargo metadata` 拿独立 workspace 的 `target_directory`，再拼出 `wasm32-wasip1/release/spec_test_guest.wasm`。 +fn guest_wasm_path() -> PathBuf { + let manifest = guest_manifest_path(); + let out = std::process::Command::new(env!("CARGO")) + .args(["metadata", "--no-deps", "--format-version", "1", "--manifest-path"]) + .arg(&manifest) + .output() + .expect("cargo metadata: spawn"); + assert!(out.status.success(), "cargo metadata failed: {}", String::from_utf8_lossy(&out.stderr)); + let meta: serde_json::Value = serde_json::from_slice(&out.stdout).expect("parse cargo metadata json"); + let target_dir = meta["target_directory"].as_str().expect("target_directory missing"); + PathBuf::from(target_dir).join("wasm32-wasip1").join("release").join("spec_test_guest.wasm") +} + +fn ensure_guest_built() -> PathBuf { + let wasm = guest_wasm_path(); + if !wasm.exists() { + let manifest = guest_manifest_path(); + eprintln!("[spec_compliance] guest wasm not found at {wasm:?}; running `cargo build --release` for spec_test_guest"); + let status = std::process::Command::new(env!("CARGO")) + .args(["build", "--release", "--target", "wasm32-wasip1", "--manifest-path"]) + .arg(&manifest) + .status() + .expect("cargo build: spawn"); + assert!(status.success(), "spec_test_guest build failed (exit = {status:?})"); + assert!(wasm.exists(), "spec_test_guest.wasm still missing after build: {wasm:?}"); + } + wasm +} + +// ───────────────────────────────────────────────────────── +// 测试 harness：直接搭一个不走 Vm 的 store/linker/instance +// ───────────────────────────────────────────────────────── + +struct GuestVm { + store: Store, + instance: Instance, +} + +impl GuestVm { + fn new(wasm_bytes: &[u8], cfg: WasmPluginShellConfig, configuration: Vec) -> Self { + let engine = shared_engine(); + let module = Module::new(engine, wasm_bytes).expect("Module::new"); + + let mut host = HostState::new(Arc::new(cfg)); + host.configuration = configuration; + // 预置一个 HTTP context，方便头部 / send_local_response 类场景。 + let ctx = RequestContext { + parent_id: host.root_context_id, + stage: ContextStage::RequestHeaders, + request_pseudo: PseudoHeaders { + method: "POST".into(), + path: "/spec".into(), + authority: "spec.local".into(), + scheme: "http".into(), + }, + request_headers: HeaderMap::new(), + ..Default::default() + }; + host.contexts.insert(HTTP_CONTEXT_ID, ctx); + host.effective_context = HTTP_CONTEXT_ID; + + let mut store: Store = Store::new(engine, host); + let mut linker: Linker = Linker::new(engine); + // dispatch_tx 在本测试里不会被消费——保留 rx 不让通道关闭即可。 + let (dispatch_tx, _dispatch_rx) = tokio::sync::mpsc::unbounded_channel::<(u32, HttpCallResult)>(); + register_all(&mut linker, dispatch_tx).expect("register_all"); + register_wasi_stubs(&mut linker).expect("register_wasi_stubs"); + + let instance = linker.instantiate(&mut store, &module).expect("instantiate"); + let mem = instance.get_memory(&mut store, "memory").expect("memory export"); + store.data_mut().memory = Some(mem); + if let Ok(a) = instance.get_typed_func::(&mut store, "proxy_on_memory_allocate") { + store.data_mut().alloc = Some(a); + } else if let Ok(a) = instance.get_typed_func::(&mut store, "malloc") { + store.data_mut().alloc = Some(a); + } else { + panic!("guest exports neither proxy_on_memory_allocate nor malloc"); + } + + // _initialize 优先（SDK 在 wasm32-wasip1 上默认导这个），回退 _start。 + if let Ok(init) = instance.get_typed_func::<(), ()>(&mut store, "_initialize") { + init.call(&mut store, ()).expect("_initialize"); + } else if let Ok(start) = instance.get_typed_func::<(), ()>(&mut store, "_start") { + start.call(&mut store, ()).expect("_start"); + } + + GuestVm { store, instance } + } + + fn run_test(&mut self, scenario: u32) -> u32 { + let f: TypedFunc = self + .instance + .get_typed_func(&mut self.store, "__run_test") + .expect("__run_test export"); + f.call(&mut self.store, scenario).expect("__run_test trap-free") + } + + fn data(&self) -> &HostState { + self.store.data() + } +} + +// ───────────────────────────────────────────────────────── +// 唯一一个 `#[test]` —— 跑完所有 scenario；隔离 shared/queue/metric 已通过 scenario 内独立 key 实现。 +// ───────────────────────────────────────────────────────── + +#[test] +fn proxy_wasm_spec_v0_2_1_compliance() { + // 准备 wasm。 + let wasm_path = ensure_guest_built(); + let wasm_bytes = std::fs::read(&wasm_path).expect("read guest wasm"); + + // 业务侧配置：plugin_name 用于 well-known property 校验；configuration 走 buffer 通道。 + let cfg = WasmPluginShellConfig { + url: format!("file://{}", wasm_path.display()), + plugin_config: serde_json::Value::Null, + plugin_name: "spec-test-plugin".to_string(), + plugin_root_id: "spec-test-root".to_string(), + plugin_vm_id: "default".to_string(), + clusters: HashMap::new(), + ..Default::default() + }; + let configuration = b"spec-test-config".to_vec(); + + let mut vm = GuestVm::new(&wasm_bytes, cfg, configuration); + + // 依次跑场景；每个返回 0 视为通过。 + let scenarios: &[(u32, &str)] = &[ + (1, "shared_data CAS roundtrip"), + (2, "shared_queue lifecycle"), + (3, "metric counter increment-only"), + (4, "metric gauge bidirectional + record"), + (5, "user property set/get"), + (6, "well-known plugin_name property"), + (7, "get_log_level"), + (8, "get_current_time_nanoseconds"), + (9, "continue_stream(HTTP_REQUEST)"), + (10, "close_stream(DOWNSTREAM) → Unimplemented"), + (11, "set_effective_context(invalid) → BadArgument"), + (12, "grpc_call → Unimplemented"), + (13, "foreign_function → NotFound"), + (14, "request `:method` pseudo header"), + (15, "add/replace/remove header"), + (16, "get_buffer(PluginConfiguration)"), + (17, "send_local_response"), + (18, "proxy_done without awaiting → NotFound"), + (19, "log at all levels"), + (20, "set_tick_period"), + ]; + + let mut failures: Vec<(u32, &str, u32)> = Vec::new(); + for &(id, name) in scenarios { + // 每个 scenario 重置一次 effective_context（部分 scenario 会改它） + vm.store.data_mut().effective_context = HTTP_CONTEXT_ID; + let code = vm.run_test(id); + if code != 0 { + failures.push((id, name, code)); + } + } + + assert!(failures.is_empty(), "spec compliance failures: {failures:?}"); + + // 额外的 host 侧副作用断言： + let st = vm.data(); + + // (17) send_local_response 应写入 ctx.local_response + let ctx = st.contexts.get(&HTTP_CONTEXT_ID).expect("http ctx present"); + let lr = ctx.local_response.as_ref().expect("local_response written by guest"); + assert_eq!(lr.status, 418, "local_response.status"); + assert_eq!(lr.body, Bytes::from_static(b"local body"), "local_response.body"); + let x_spec = lr + .headers + .get("x-spec") + .expect("x-spec header present") + .to_str() + .unwrap_or(""); + assert_eq!(x_spec, "teapot"); + + // (20) set_tick_period 应写入 HostState.tick_period_ms + assert_eq!(st.tick_period_ms, Some(123), "tick_period_ms"); + + // (5) user_properties 应记录我们设的 key + // user_properties 的 key 是 path 用 \0 拼起来；spec_test_guest 设的是 vec!["spec","user_prop"] + let want_key: Vec = b"spec\0user_prop".to_vec(); + let v = st.user_properties.get(&want_key).expect("user_prop stored under '\\0'-joined key"); + assert_eq!(v.as_slice(), b"hello"); +} diff --git a/crates/plugin-wasm/tests/spec_test_guest/.cargo/config.toml b/crates/plugin-wasm/tests/spec_test_guest/.cargo/config.toml new file mode 100644 index 00000000..6b509f5b --- /dev/null +++ b/crates/plugin-wasm/tests/spec_test_guest/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +target = "wasm32-wasip1" diff --git a/crates/plugin-wasm/tests/spec_test_guest/Cargo.toml b/crates/plugin-wasm/tests/spec_test_guest/Cargo.toml new file mode 100644 index 00000000..a0b8e23d --- /dev/null +++ b/crates/plugin-wasm/tests/spec_test_guest/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "spec_test_guest" +version = "0.0.0" +edition = "2021" +publish = false +description = "Proxy-Wasm guest used by spacegate-plugin-wasm integration tests. Mirrors proxy-wasm-rust-sdk examples and exercises every host fn our host registers." + +# 独立 workspace：本 crate 目标 `wasm32-wasip1`，不参与外层 host workspace。 +[workspace] + +[lib] +crate-type = ["cdylib"] + +[dependencies] +proxy-wasm = "0.2" +log = "0.4" + +[profile.release] +codegen-units = 1 +opt-level = "z" +lto = "fat" +strip = true +panic = "abort" diff --git a/crates/plugin-wasm/tests/spec_test_guest/src/lib.rs b/crates/plugin-wasm/tests/spec_test_guest/src/lib.rs new file mode 100644 index 00000000..cd5a1b99 --- /dev/null +++ b/crates/plugin-wasm/tests/spec_test_guest/src/lib.rs @@ -0,0 +1,338 @@ +//! 验证 spacegate-plugin-wasm host fn 实现的 **真实 proxy-wasm guest 插件**。 +//! +//! 用法：通过 `cargo build --release` 编译到 `wasm32-wasip1`，得到 +//! `target/wasm32-wasip1/release/spec_test_guest.wasm`，再由 +//! `crates/plugin-wasm/tests/spec_compliance.rs` 加载并依次调用 +//! [`__run_test`] 来跑各场景。 +//! +//! 设计取舍：每个场景都通过 [`proxy_wasm::hostcalls`] 直接调相应 host fn。 +//! SDK 在 status 不预期时会 panic（即 wasmtime trap），这正好让我们： +//! +//! - host fn 返回正确 Status → SDK 返回 Result → guest 自行断言并返回 0 / 失败码 +//! - host fn 返回错误 Status → SDK panic → wasmtime trap → test 立刻挂掉 +//! +//! 这样测试侧只看 `__run_test` 返回值就能判定通过。 + +use std::time::Duration; + +use proxy_wasm::hostcalls; +use proxy_wasm::traits::*; +use proxy_wasm::types::*; + +proxy_wasm::main! {{ + proxy_wasm::set_log_level(LogLevel::Trace); + proxy_wasm::set_root_context(|_| -> Box { Box::new(SpecRoot) }); +}} + +struct SpecRoot; + +impl Context for SpecRoot {} +impl RootContext for SpecRoot { + fn on_vm_start(&mut self, _vm_configuration_size: usize) -> bool { true } + fn on_configure(&mut self, _plugin_configuration_size: usize) -> bool { true } + fn get_type(&self) -> Option { Some(ContextType::HttpContext) } + fn create_http_context(&self, _context_id: u32) -> Option> { + Some(Box::new(SpecHttp)) + } +} + +struct SpecHttp; +impl Context for SpecHttp {} +impl HttpContext for SpecHttp {} + +// ───────────────────────────────────────────────────────── +// 直接 extern：spec 里这些 host fn SDK 没暴露，我们手动 import 用 +// ───────────────────────────────────────────────────────── + +extern "C" { + fn proxy_grpc_call( + a: *const u8, b: usize, c: *const u8, d: usize, e: *const u8, f: usize, + g: *const u8, h: usize, i: *const u8, j: usize, k: u32, l: *mut u32, + ) -> u32; + fn proxy_call_foreign_function( + a: *const u8, b: usize, c: *const u8, d: usize, e: *mut *mut u8, f: *mut usize, + ) -> u32; + fn proxy_continue_stream(stream_type: u32) -> u32; + fn proxy_close_stream(stream_type: u32) -> u32; + fn proxy_set_effective_context(ctx: u32) -> u32; + fn proxy_done() -> u32; +} + +// spec §Types: STATUS_UNIMPLEMENTED = 12, STATUS_NOT_FOUND = 1, STATUS_BAD_ARGUMENT = 2 +const STATUS_OK: u32 = 0; +const STATUS_NOT_FOUND: u32 = 1; +const STATUS_BAD_ARGUMENT: u32 = 2; +const STATUS_UNIMPLEMENTED: u32 = 12; + +const STREAM_HTTP_REQUEST: u32 = 0; +const STREAM_DOWNSTREAM: u32 = 2; + +// ───────────────────────────────────────────────────────── +// 测试入口 +// ───────────────────────────────────────────────────────── + +#[no_mangle] +pub extern "C" fn __run_test(scenario: u32) -> u32 { + match scenario { + 1 => test_shared_data(), + 2 => test_shared_queue(), + 3 => test_metric_counter(), + 4 => test_metric_gauge(), + 5 => test_user_property(), + 6 => test_well_known_plugin_name(), + 7 => test_log_level(), + 8 => test_current_time(), + 9 => test_continue_stream_http_request(), + 10 => test_close_stream_tcp_unimplemented(), + 11 => test_set_effective_context_bad_argument(), + 12 => test_grpc_unimplemented(), + 13 => test_foreign_function_not_found(), + 14 => test_request_header_pseudo_method(), + 15 => test_add_replace_remove_header(), + 16 => test_get_configuration_buffer(), + 17 => test_send_local_response(), + 18 => test_done_without_pending(), + 19 => test_log(), + 20 => test_tick_period(), + _ => 999, + } +} + +// ─── 1: shared_data CAS roundtrip ─── +fn test_shared_data() -> u32 { + let key = "spec.shared_data.k"; + hostcalls::set_shared_data(key, Some(b"v1"), None).unwrap(); + let (val, cas) = hostcalls::get_shared_data(key).unwrap(); + if val.as_deref() != Some(b"v1".as_slice()) { return 1; } + let cas = match cas { Some(c) if c > 0 => c, _ => return 2 }; + // 错误 cas → CasMismatch（SDK 把 CasMismatch 包装成 Err） + if hostcalls::set_shared_data(key, Some(b"v2"), Some(cas.wrapping_add(999))).is_ok() { + return 3; + } + // 正确 cas → Ok + hostcalls::set_shared_data(key, Some(b"v2"), Some(cas)).unwrap(); + let (val, cas2) = hostcalls::get_shared_data(key).unwrap(); + if val.as_deref() != Some(b"v2".as_slice()) { return 4; } + let cas2 = cas2.unwrap_or(0); + if cas2 <= cas { return 5; } + 0 +} + +// ─── 2: shared queue lifecycle ─── +fn test_shared_queue() -> u32 { + let qid = hostcalls::register_shared_queue("spec.q.basic").unwrap(); + if qid == 0 { return 1; } + // register 同名应返回相同 qid（spec §proxy_register_shared_queue） + if hostcalls::register_shared_queue("spec.q.basic").unwrap() != qid { return 2; } + // resolve_shared_queue：vm_id 默认是 "default"，name 已存在 + match hostcalls::resolve_shared_queue("default", "spec.q.basic").unwrap() { + Some(id) if id == qid => {} + _ => return 3, + } + if hostcalls::resolve_shared_queue("default", "spec.q.nonexistent").unwrap().is_some() { return 4; } + hostcalls::enqueue_shared_queue(qid, Some(b"a")).unwrap(); + hostcalls::enqueue_shared_queue(qid, Some(b"bb")).unwrap(); + match hostcalls::dequeue_shared_queue(qid).unwrap() { + Some(v) if v == b"a".to_vec() => {} + _ => return 5, + } + match hostcalls::dequeue_shared_queue(qid).unwrap() { + Some(v) if v == b"bb".to_vec() => {} + _ => return 6, + } + // 空 → Ok(None)（SDK 把 Empty 折叠成 None） + if hostcalls::dequeue_shared_queue(qid).unwrap().is_some() { return 7; } + // 未知 qid → Err(NotFound) + if hostcalls::dequeue_shared_queue(9_999_999).is_ok() { return 8; } + if hostcalls::enqueue_shared_queue(9_999_999, Some(b"x")).is_ok() { return 9; } + 0 +} + +// ─── 3: counter only allows positive delta ─── +fn test_metric_counter() -> u32 { + let id = hostcalls::define_metric(MetricType::Counter, "spec.counter").unwrap(); + if id == 0 { return 1; } + hostcalls::increment_metric(id, 3).unwrap(); + hostcalls::increment_metric(id, 2).unwrap(); + if hostcalls::get_metric(id).unwrap() != 5 { return 2; } + // counter 不能 decrement → BadArgument + if hostcalls::increment_metric(id, -1).is_ok() { return 3; } + if hostcalls::get_metric(id).unwrap() != 5 { return 4; } + // 未知 mid → NotFound + if hostcalls::get_metric(9_999_999).is_ok() { return 5; } + 0 +} + +// ─── 4: gauge bidirectional + record ─── +fn test_metric_gauge() -> u32 { + let id = hostcalls::define_metric(MetricType::Gauge, "spec.gauge").unwrap(); + hostcalls::increment_metric(id, 10).unwrap(); + hostcalls::increment_metric(id, -3).unwrap(); + if hostcalls::get_metric(id).unwrap() != 7 { return 1; } + hostcalls::record_metric(id, 42).unwrap(); + if hostcalls::get_metric(id).unwrap() != 42 { return 2; } + 0 +} + +// ─── 5: user property set/get roundtrip ─── +fn test_user_property() -> u32 { + let path = vec!["spec", "user_prop"]; + hostcalls::set_property(path.clone(), Some(b"hello")).unwrap(); + let v = hostcalls::get_property(path.clone()).unwrap(); + if v.as_deref() != Some(b"hello".as_slice()) { return 1; } + // None / NotFound：未设置过的 path + let missing = hostcalls::get_property(vec!["spec", "absent"]).unwrap(); + if missing.is_some() { return 2; } + 0 +} + +// ─── 6: well-known property plugin_name ─── +fn test_well_known_plugin_name() -> u32 { + let v = hostcalls::get_property(vec!["plugin_name"]).unwrap(); + match v { + Some(b) if b == b"spec-test-plugin".to_vec() => 0, + Some(_) => 1, + None => 2, + } +} + +// ─── 7: log_level（host 当前 tracing 最大级别） ─── +fn test_log_level() -> u32 { + let lvl = hostcalls::get_log_level().unwrap(); + // host 默认 tracing 是 ERROR 以上；我们的实现至少返回 5（CRITICAL）或更宽 + // 只要不 panic 且能拿到值就算 OK + let _ = lvl; + 0 +} + +// ─── 8: current_time > 0 ─── +fn test_current_time() -> u32 { + let now = hostcalls::get_current_time().unwrap(); + if now < std::time::UNIX_EPOCH { return 1; } + 0 +} + +// ─── 9: continue_stream(HTTP_REQUEST) → Ok ─── +fn test_continue_stream_http_request() -> u32 { + let s = unsafe { proxy_continue_stream(STREAM_HTTP_REQUEST) }; + if s != STATUS_OK { return s; } + 0 +} + +// ─── 10: close_stream(DOWNSTREAM) → Unimplemented（TCP 我们不支持） ─── +fn test_close_stream_tcp_unimplemented() -> u32 { + let s = unsafe { proxy_close_stream(STREAM_DOWNSTREAM) }; + if s != STATUS_UNIMPLEMENTED { return 100 + s; } + 0 +} + +// ─── 11: set_effective_context 对未知 ctx → BadArgument ─── +fn test_set_effective_context_bad_argument() -> u32 { + let s = unsafe { proxy_set_effective_context(987654) }; + if s != STATUS_BAD_ARGUMENT { return 100 + s; } + 0 +} + +// ─── 12: gRPC host fn → Unimplemented ─── +fn test_grpc_unimplemented() -> u32 { + let mut tok: u32 = 0; + let s = unsafe { + proxy_grpc_call( + b"cluster".as_ptr(), 7, + b"svc".as_ptr(), 3, + b"m".as_ptr(), 1, + std::ptr::null(), 0, + std::ptr::null(), 0, + 1000, + &mut tok as *mut u32, + ) + }; + if s != STATUS_UNIMPLEMENTED { return 100 + s; } + 0 +} + +// ─── 13: foreign_function → NotFound（无注册表） ─── +fn test_foreign_function_not_found() -> u32 { + let mut data: *mut u8 = std::ptr::null_mut(); + let mut size: usize = 0; + let s = unsafe { + proxy_call_foreign_function( + b"some_fn".as_ptr(), 7, + b"args".as_ptr(), 4, + &mut data as *mut *mut u8, + &mut size as *mut usize, + ) + }; + if s != STATUS_NOT_FOUND { return 100 + s; } + 0 +} + +// ─── 14: get_http_request_header(":method") ─── +fn test_request_header_pseudo_method() -> u32 { + match hostcalls::get_map_value(MapType::HttpRequestHeaders, ":method").unwrap() { + Some(m) if m == "POST" => 0, + Some(_) => 1, + None => 2, + } +} + +// ─── 15: add / replace / remove header on HttpRequestHeaders ─── +fn test_add_replace_remove_header() -> u32 { + hostcalls::add_map_value(MapType::HttpRequestHeaders, "x-spec-add", "v1").unwrap(); + if hostcalls::get_map_value(MapType::HttpRequestHeaders, "x-spec-add").unwrap().as_deref() != Some("v1") { + return 1; + } + // SDK 用 set_map_value(map, key, Some("v2")) 触发 spec 的 replace 语义。 + hostcalls::set_map_value(MapType::HttpRequestHeaders, "x-spec-add", Some("v2")).unwrap(); + if hostcalls::get_map_value(MapType::HttpRequestHeaders, "x-spec-add").unwrap().as_deref() != Some("v2") { + return 2; + } + hostcalls::remove_map_value(MapType::HttpRequestHeaders, "x-spec-add").unwrap(); + if hostcalls::get_map_value(MapType::HttpRequestHeaders, "x-spec-add").unwrap().is_some() { + return 3; + } + 0 +} + +// ─── 16: get_buffer(PluginConfiguration) 返回配置字节 ─── +fn test_get_configuration_buffer() -> u32 { + // start=0, max_size=usize::MAX + match hostcalls::get_buffer(BufferType::PluginConfiguration, 0, usize::MAX).unwrap() { + Some(b) if b == b"spec-test-config".to_vec() => 0, + Some(_) => 1, + None => 2, + } +} + +// ─── 17: send_local_response（host 侧通过 contexts[ctx].local_response 验证） ─── +fn test_send_local_response() -> u32 { + hostcalls::send_http_response( + 418, + vec![("x-spec", "teapot")], + Some(b"local body"), + ).unwrap(); + 0 +} + +// ─── 18: proxy_done 在没有 awaiting_done 时 → NotFound ─── +fn test_done_without_pending() -> u32 { + let s = unsafe { proxy_done() }; + if s != STATUS_NOT_FOUND { return 100 + s; } + 0 +} + +// ─── 19: proxy_log 在各级别 ─── +fn test_log() -> u32 { + hostcalls::log(LogLevel::Trace, "spec trace").unwrap(); + hostcalls::log(LogLevel::Debug, "spec debug").unwrap(); + hostcalls::log(LogLevel::Info, "spec info").unwrap(); + hostcalls::log(LogLevel::Warn, "spec warn").unwrap(); + hostcalls::log(LogLevel::Error, "spec error").unwrap(); + 0 +} + +// ─── 20: set_tick_period 应 Ok ─── +fn test_tick_period() -> u32 { + hostcalls::set_tick_period(Duration::from_millis(123)).unwrap(); + 0 +} diff --git a/docs/CODE_REVIEW.md b/docs/CODE_REVIEW.md new file mode 100644 index 00000000..7483d97a --- /dev/null +++ b/docs/CODE_REVIEW.md @@ -0,0 +1,222 @@ +# Spacegate 代码审核报告 + +> 文档生成日期：2026-05-12 +> 范围：整个 Spacegate workspace（`spacegate-kernel` / `spacegate-plugin` / `spacegate-model` / `spacegate-config` / `spacegate-shell` / extensions / binaries / SDK） + +本文档对仓库整体架构、各 crate 功能点、亮点及风险进行归纳，便于评审与后续修复跟踪。 + +--- + +## 一、项目总览 + +Spacegate 是基于 Rust 与 hyper 的 **库优先（library-first）** API 网关，强调云原生（Kubernetes Gateway API）与插件扩展。`Cargo.toml` 中 workspace 成员包含： + +- **二进制**：`binary/spacegate`、`binary/admin-server` +- **核心库**：`crates/kernel`、`crates/plugin`、`crates/model`、`crates/config`、`crates/shell` +- **扩展**：`crates/extension/axum`、`crates/extension/redis` +- **示例**：`examples/sayhello`、`examples/socks5-proxy`、`examples/mitm-proxy` 等 + +分层关系（自下而上）： + +| 层 | crate | 职责 | +|----|--------|------| +| 数据模型 | `spacegate-model` | 网关/路由/插件/匹配规则 DTO，可选 ts-rs 导出 | +| 配置后端 | `spacegate-config` | 文件 / K8s / Redis / 内存：CRUD + 事件监听 | +| 核心运行时 | `spacegate-kernel` | TCP 监听、HTTPS、路由匹配、Backend、helper layer | +| 扩展库 | `spacegate-ext-axum`、`spacegate-ext-redis` | 全局 axum 服务、Redis 客户端仓库 | +| 插件系统 | `spacegate-plugin` | Plugin trait、动态库、内置插件、挂载点 | +| 集成入口 | `spacegate-shell` | 配置与内核映射、热更新、生命周期 | +| 二进制 | `spacegate`、`spacegate-admin-server` | 网关进程、管理后台 | +| SDK | `sdk/admin-client` | TypeScript，对接 admin-server | + +--- + +## 二、`spacegate-kernel` + +### 2.1 主要功能 + +1. **TCP 监听与协议嗅探**：`SgListen` 通过 `peek` 后由 `TcpService::sniff` 选择 HTTP/HTTPS/SOCKS5 等。 +2. **HTTP/1.1、HTTP/2、WebSocket、HTTPS**：`Http`/`Https` 实现 `TcpService`；`HyperServiceAdapter` 将请求转为 `SgBody` 并注入 `PeerAddr`、`EnterTime`、`Reflect`。 +3. **网关装配**：`http_gateway::Gateway`（builder）含网关级插件链、`HttpRoute` 表、`Reloader` 支持热更新路由。 +4. **主机名匹配**：`HostnameTree`（`match_hostname.rs`），支持 IPv4/IPv6、通配域名、优先级排序。 +5. **路由匹配**：`HttpRouteMatch` 支持 path（Exact/Prefix/Regex）、headers、query、method；多重 match 在单条规则内为 AND；`Vec` 层为 OR。 +6. **后端**：`http_backend_service`（`x-forwarded-for`、WebSocket 升级与双向拷贝）、`static_file_service`、全局 `ClientRepo` 与可插拔 `HttpClient`。 +7. **辅助层**：`TimeoutLayer`、`ReloadLayer`（`ShardedLock`）、`Balancer`（`IpHash` / 加权随机）、`MapRequest`/`MapFuture`、`RouterService`。 +8. **扩展与工具**：`Reflect`、`Defer`、`OriginalIpAddr`、`MatchedSgRouter`、`Authorization`、`SgBody`（dump 后可克隆）。 + +### 2.2 亮点 + +- `BoxLayer` 与 tower 组合良好，网关/路由/规则/后端多级挂载清晰。 +- `Reloader` + `OnceLock` 读多写少场景友好。 +- `HostnameTree` 设计文档与测试较完整。 + +### 2.3 风险与问题 + +- **TLS 客户端默认跳过服务端证书校验**：`ClientRepo::default` 使用 `get_rustls_config_dangerous`；`SgParameters::ignore_tls_verification` 在代码中未见实质接线。**生产环境风险高**，建议默认走系统根证书，仅显式配置才关闭校验。 +- **静态文件路径规范化**：`canonicalize` 失败时回退到 `dir`，可能削弱「必须在目录下」的语义，建议失败即 404。 +- **`HttpBackendService` 使用 `unwrap_unchecked`**：可改为显式处理以更清晰。 +- **`create_http_router` 中 hostname 索引**：当 `route.hostnames` 非空时，新建节点误落到 `"*"` 的逻辑需核对是否为 bug（应绑定具体 hostname）。 +- **`SgBody::clone` 未 dump 会 panic**：插件作者易踩坑，需在文档中突出。 +- **方法匹配注释与 `Vec` 的 OR 语义**：注释若写「仅当指定 method」易与实现不一致。 + +--- + +## 三、`spacegate-plugin` + +### 3.1 主要功能 + +1. **Plugin trait**：`CODE`、`call`、`create`，可选 `MONO`、`schema_opt`、元数据。 +2. **`PluginRepository`**：全局注册表、实例 CRUD、`register_dylib`、快照与挂载追踪。 +3. **`PluginInstance`**：`ArcSwap` 热替换函数、生命周期钩子、`DropTracer` 防悬挂挂载索引。 +4. **挂载点**：网关 / 路由 / 规则 / 后端四级 `MountPointIndex`。 +5. **内置插件（按 feature）**：如 `static-resource`、`limit`（Redis Lua）、`header-modifier`、`redirect`、`rewrite`、`set-version`、`set-scheme`、`maintenance`、`inject`、`east-west-traffic-white-list`，以及 Redis 系列（`redis-count`、`redis-limit`、`redis-time-range`、`redis-dynamic-route`）。 + +### 3.2 亮点 + +- `PluginError` 统一错误响应与 `X-Plugin-Error` 头。 +- 部分 Redis 插件含 testcontainers 集成测试。 + +### 3.3 风险与问题 + +- **`redirect` 插件**：解析 URL 后未真正返回 3xx 或未改写请求，接近 no-op，需补全实现。 +- **遗留/禁用模块**：`breaker.rs` 空文件；`decompression`/`status`/`retry` 等与旧 API 耦合且未在 `register_prelude` 启用，建议清理或重写。 +- **`SystemTime::now().duration_since(UNIX_EPOCH).expect(...)`**：时间异常时可能 panic。 +- **仓库锁**：`RwLock` + 多层 `expect`；钩子里若再次操作仓库可能死锁，需在文档约束。 +- **`reflect` 扩展缺失会 panic**：非标准入口构造的请求需注意。 +- **`FromBackend::unsafe new` 用法**：可与实际调用路径再核对是否必须 unsafe。 + +--- + +## 四、`spacegate-model` + +### 4.1 主要功能 + +- `SgGateway`、`SgHttpRoute`、`SgBackendRef`、`BackendHost`、`PluginInstanceId/Name`、`PluginConfig`、`PluginInstanceMap`。 +- 可选 `typegen`（ts-rs）供前端/SDK。 +- K8s 相关扩展（CRD 等）。 + +### 4.2 风险与问题 + +- **`PluginInstanceName` 的 `Display` 与 `FromStr` 不一致**：Mono 显示为 `m` 而解析期望 `g`，可能影响依赖字符串往返的配置/通道。 +- **`PluginInstanceMap` 反序列化**：错误路径使用 `eprintln!`，建议改为 `tracing`。 + +--- + +## 五、`spacegate-config` + +### 5.1 主要功能 + +- Trait：`Create`、`Retrieve`、`Update`、`Delete`；`CreateListener` + `Listen`；`ConfigType` / `ConfigEventType`。 +- **实现**：`Memory`（静态）、`Fs`（目录布局 + Unix SIGHUP / Windows notify）、`K8s`（多资源 watch + SIGHUP 全局重载）、`Redis`（hash + pubsub）。 +- **Discovery**：实例列表与可选后端发现（如 fs 下读 `/var/www`）。 + +### 5.2 风险与问题 + +- **K8s 路由事件**：`process_http_spaceroute_event` 中 Applied 与 Delete 的事件类型是否应区分 Update/Delete，需与 shell 中「全量拉路由」行为对照，避免语义混淆。 +- **监听任务中 `send(...).expect`**：通道关闭会导致 panic。 +- **`Fs::modify_cached` 全目录删建**：中断可能丢配置，宜加备份或原子写。 +- **`redis/listen.rs` 中未使用的 `CHANGE_CACHE`**：死代码。 +- **`RedisListener::CONFIG_LISTENER_NAME` 误写为 `"file"`**：应为 `"redis"` 以免日志误导。 + +--- + +## 六、`spacegate-shell` + +### 6.1 主要功能 + +- `startup_file` / `startup_k8s` / `startup_redis` / `startup_static` → 统一 `startup`。 +- `RunningSgGateway`：`global_init`、`global_reset`、`global_update`（`Reloader` 热更路由）。 +- 配置到内核：`collect_http_route`、`global_batch_mount_plugin`、K8s Service 扩展注入。 +- 启用 `ext-axum` 时：健康检查、`/control/push_event`、静态页等。 + +### 6.2 风险与问题 + +- **Route 类事件**：handler 对 Create/Update/Delete 一律 `retrieve_config_item_all_routes` 后整体更新，语义依赖「全量正确」；与 K8s 事件类型需一起审视。 +- **插件初始化失败**：当前多为日志后继续，可按策略支持 fail-fast。 +- **全局 `Mutex` 中毒**：`expect("poisoned lock")` 后难以恢复。 +- **TLS `enable_secret_extraction`**：生产宜可配置关闭。 + +--- + +## 七、扩展库 + +### `spacegate-ext-redis` + +- `RedisClient::get_conn` / `From<&str>` 使用 `unwrap`/`expect`，配置错误易 panic；建议提供 `try_*` API。 + +### `spacegate-ext-axum` + +- `GlobalAxumServer` 关停路径存在 `expect`；`InternalError` 里有 `unwrap` 组 Response。 + +--- + +## 八、`binary/spacegate` + +- Clap 参数：`file:`/`k8s:`/`redis:`/`static:`；可选动态库目录扫描加载。 +- 缺 feature 时 dylib 仅 `eprintln`，建议统一 tracing。 + +--- + +## 九、`binary/admin-server` + +### 主要功能 + +- `/config/*`、`/plugin/*`、`/auth/login`、`/discovery/*`。 +- JWT + 可选 SK 摘要；`X-Client-Version` / `X-Server-Version` 乐观并发。 +- 发现：实例健康、插件列表/schema 缓存、向网关 `push_event` 触发重载。 + +### 风险与问题 + +- **空文件**：`mw/instance_select.rs` 等遗留。 +- **跨平台**：`clap` 等处 `unix` 专有 import/默认值未守卫时 Windows 编译可能失败。 +- **健康检查与 `sync_attr_cache` 缓存**：若使用 `Instant::elapsed() >= Duration::ZERO` 判断是否过期，逻辑恒为「已过期」，缓存失效——应改为与 `Instant::now()` 比较。 +- **依赖版本**：如 `tower-http` 与 workspace 不一致可能导致重复编译。 +- **未配置鉴权时中间件放行**：部署文档需强调必须配置密钥。 + +--- + +## 十、`sdk/admin-client` + +- Axios 封装，与 admin-server API 对齐；版本冲突与 401 自定义异常。 +- 注意全局 client 与 `clientVersion` 刷新页丢失导致的首次 409。 + +--- + +## 十一、示例 + +- **sayhello**：动态库插件最小示例。 +- **socks5-proxy**：`TcpService` + 端口多协议嗅探。 +- **mitm-proxy**：CONNECT + 动态证书 MITM 演示。 + +--- + +## 十二、横向问题汇总 + +| 优先级 | 问题 | +|--------|------| +| 高 | TLS 默认信任任意后端证书；`ignore_tls_verification` 未接线 | +| 高 | `redirect` 插件未真正重定向 | +| 高 | `GatewayRouter` hostname 索引与通配 `*` 的逻辑需复核 | +| 高 | K8s 监听中路由事件类型与 shell 全量更新语义 | +| 高 | `PluginInstanceName` Display/FromStr 不一致 | +| 高 | admin-server 健康/attr 缓存时间判断错误 | +| 中 | 大量 `unwrap`/`expect`；Redis/Axum 扩展 panic 路径 | +| 中 | Windows 编译（unix-only 模块） | +| 中 | 死代码与误填常量（如 Redis listener 名称） | +| 低 | tracing 替代 eprintln；依赖版本对齐 | + +--- + +## 十三、建议修复顺序（供迭代跟踪） + +1. **安全与正确性**：TLS 默认策略、`redirect`、hostname 路由索引、K8s 事件语义、PluginInstanceName 往返、admin-server 缓存判断。 +2. **健壮性**：减少 expect；Redis `try_get`；插件钩子使用规范文档。 +3. **可维护性**：清理 breaker/status 等废弃路径；统一 tower-http 版本；修正 `CONFIG_LISTENER_NAME`。 + +--- + +## 十四、修订历史 + +| 日期 | 说明 | +|------|------| +| 2026-05-12 | 初版：基于全仓库结构与关键源码路径的审核汇总 | From 5bc72b67aae298d1c54e6b9d4877cf95beb0989b Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Thu, 14 May 2026 18:27:39 +0800 Subject: [PATCH 03/19] Add wasm hello world demo plugin --- examples/wasm-hello/Cargo.toml | 23 ++++++++ examples/wasm-hello/README.md | 31 +++++++++++ examples/wasm-hello/src/lib.rs | 52 ++++++++++++++++++ resource/wasm-hello-demo/config.json | 5 ++ .../gateway/wasm-hello/config.json | 23 ++++++++ .../gateway/wasm-hello/route/hello.json | 26 +++++++++ .../plugin/wasm.hello-world.json | 10 ++++ resource/wasm/spacegate_wasm_hello.wasm | Bin 0 -> 86567 bytes 8 files changed, 170 insertions(+) create mode 100644 examples/wasm-hello/Cargo.toml create mode 100644 examples/wasm-hello/README.md create mode 100644 examples/wasm-hello/src/lib.rs create mode 100644 resource/wasm-hello-demo/config.json create mode 100644 resource/wasm-hello-demo/gateway/wasm-hello/config.json create mode 100644 resource/wasm-hello-demo/gateway/wasm-hello/route/hello.json create mode 100644 resource/wasm-hello-demo/plugin/wasm.hello-world.json create mode 100755 resource/wasm/spacegate_wasm_hello.wasm diff --git a/examples/wasm-hello/Cargo.toml b/examples/wasm-hello/Cargo.toml new file mode 100644 index 00000000..58a93142 --- /dev/null +++ b/examples/wasm-hello/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "spacegate_wasm_hello" +version = "0.0.0" +edition = "2021" +publish = false +description = "Minimal Proxy-Wasm hello world plugin for Spacegate." + +# Standalone workspace: this crate builds for wasm32-wasip1 and is not part of +# the host Spacegate workspace. +[workspace] + +[lib] +crate-type = ["cdylib"] + +[dependencies] +proxy-wasm = "0.2" + +[profile.release] +codegen-units = 1 +opt-level = "z" +lto = "fat" +strip = true +panic = "abort" diff --git a/examples/wasm-hello/README.md b/examples/wasm-hello/README.md new file mode 100644 index 00000000..264e5f2a --- /dev/null +++ b/examples/wasm-hello/README.md @@ -0,0 +1,31 @@ +# Spacegate Wasm Hello World + +This is a minimal Proxy-Wasm guest plugin for Spacegate. + +Build the wasm: + +```bash +cd examples/wasm-hello +cargo build --release --target wasm32-wasip1 +cd ../.. +cp examples/wasm-hello/target/wasm32-wasip1/release/spacegate_wasm_hello.wasm resource/wasm/spacegate_wasm_hello.wasm +``` + +Run Spacegate with the demo config from the repository root: + +```bash +RUST_LOG=info cargo run -p spacegate --features wasm -- -c file:resource/wasm-hello-demo +``` + +On startup, Spacegate should log: + +```text +hello world from spacegate wasm plugin +hello world wasm plugin configured +``` + +The demo route also lets the plugin return a direct response: + +```bash +curl http://127.0.0.1:18082/hello-world +``` diff --git a/examples/wasm-hello/src/lib.rs b/examples/wasm-hello/src/lib.rs new file mode 100644 index 00000000..bbdda714 --- /dev/null +++ b/examples/wasm-hello/src/lib.rs @@ -0,0 +1,52 @@ +use proxy_wasm::hostcalls; +use proxy_wasm::traits::*; +use proxy_wasm::types::*; + +const HELLO: &str = "hello world from spacegate wasm plugin"; + +proxy_wasm::main! {{ + proxy_wasm::set_log_level(LogLevel::Info); + proxy_wasm::set_root_context(|_| -> Box { Box::new(HelloRoot) }); +}} + +struct HelloRoot; + +impl Context for HelloRoot {} + +impl RootContext for HelloRoot { + fn on_vm_start(&mut self, _: usize) -> bool { + let _ = hostcalls::log(LogLevel::Info, HELLO); + true + } + + fn on_configure(&mut self, _: usize) -> bool { + let _ = hostcalls::log(LogLevel::Info, "hello world wasm plugin configured"); + true + } + + fn create_http_context(&self, _: u32) -> Option> { + Some(Box::new(HelloHttp)) + } + + fn get_type(&self) -> Option { + Some(ContextType::HttpContext) + } +} + +struct HelloHttp; + +impl Context for HelloHttp {} + +impl HttpContext for HelloHttp { + fn on_http_request_headers(&mut self, _: usize, _: bool) -> Action { + let _ = hostcalls::log(LogLevel::Info, "hello world request reached wasm plugin"); + self.add_http_request_header("x-wasm-hello", "hello-world"); + + if self.get_http_request_header(":path").as_deref() == Some("/hello-world") { + self.send_http_response(200, vec![("content-type", "text/plain"), ("x-powered-by", "spacegate-wasm")], Some(b"hello world\n")); + return Action::Pause; + } + + Action::Continue + } +} diff --git a/resource/wasm-hello-demo/config.json b/resource/wasm-hello-demo/config.json new file mode 100644 index 00000000..20d4e030 --- /dev/null +++ b/resource/wasm-hello-demo/config.json @@ -0,0 +1,5 @@ +{ + "gateways": {}, + "plugins": {}, + "api_port": 19876 +} diff --git a/resource/wasm-hello-demo/gateway/wasm-hello/config.json b/resource/wasm-hello-demo/gateway/wasm-hello/config.json new file mode 100644 index 00000000..a751fe77 --- /dev/null +++ b/resource/wasm-hello-demo/gateway/wasm-hello/config.json @@ -0,0 +1,23 @@ +{ + "gateway": { + "name": "wasm-hello", + "parameters": {}, + "listeners": [ + { + "name": "http", + "ip": "127.0.0.1", + "port": 18082, + "protocol": { + "type": "http" + } + } + ], + "plugins": [ + { + "code": "wasm", + "kind": "named", + "name": "hello-world" + } + ] + } +} diff --git a/resource/wasm-hello-demo/gateway/wasm-hello/route/hello.json b/resource/wasm-hello-demo/gateway/wasm-hello/route/hello.json new file mode 100644 index 00000000..78e75bb4 --- /dev/null +++ b/resource/wasm-hello-demo/gateway/wasm-hello/route/hello.json @@ -0,0 +1,26 @@ +{ + "route_name": "hello", + "rules": [ + { + "matches": [ + { + "path": { + "kind": "Prefix", + "value": "/" + } + } + ], + "backends": [ + { + "host": { + "kind": "Host", + "host": "127.0.0.1" + }, + "port": 18099, + "weight": 1 + } + ] + } + ], + "priority": 0 +} diff --git a/resource/wasm-hello-demo/plugin/wasm.hello-world.json b/resource/wasm-hello-demo/plugin/wasm.hello-world.json new file mode 100644 index 00000000..b2327fb7 --- /dev/null +++ b/resource/wasm-hello-demo/plugin/wasm.hello-world.json @@ -0,0 +1,10 @@ +{ + "url": "resource/wasm/spacegate_wasm_hello.wasm", + "validate_on_create": false, + "fail_strategy": "fail_close", + "plugin_name": "hello-world", + "plugin_root_id": "hello-world-root", + "plugin_vm_id": "hello-world-vm", + "plugin_config": {}, + "clusters": {} +} diff --git a/resource/wasm/spacegate_wasm_hello.wasm b/resource/wasm/spacegate_wasm_hello.wasm new file mode 100755 index 0000000000000000000000000000000000000000..0841d0eabe48f1ad2e23656c14d9bdf518a9c21e GIT binary patch literal 86567 zcmdqK4ZNLKUFZ9}?6-6FKFL#v5=k7My@{G$+abg}Ms@uerLd){{Z9aoY5i)!E28i`&sReiN* zH|^JI)ood+Zi?o09qNx>)UQ=QOUdrt@ow7OO&?V*(VuQ8WAFFyPG9dX5MgEa?B=C# z5q|X&1`w7dtj>y=cRh zOE28GWyAJOJ1)C)>yAxPPw#&ye0b^BotJEVS@YWo|8C2rFNq?R9Tj!$+_3YK3pa1L zY}58jF1={OOE1~7<&qtnF1&Q>MLT?>KkN&>WYf+Kzxc8jzj)L34Zrw`ott(<6E#rv zl48`c^A*u#?ZuAcUSw5uJ9ci|`LZ1zoB+Bi+Hvv5?VB#zaM8w{8+{|;o$5O7-5H^} z?U!A+;lhnuFWj`nLmj%O`YwFg_U)Uts+%v}v|;PUtzfm*y?O-!*L`)MfYT}#C49Y0 z8|`h`5)D*1JiU15&dY#;r&D_RL|^uzP1|0!N!0NDvK!y&P+-&6lN5l;mu}fLR)gVx zg>OP_Zg}yf+c#bElC2wF{IabV?!4sEt|^pZ<!xU_Kna{AIv&qSp}LJ1U3B91 z^mPrOT6_P_x3_)MWm`60xT#*K2KA2?gyk|`|EWG5ykz^OTQ`8QC|RJ;8AYKTm;ColJ5E@% z_u`8-ynOp5J2yp>3sfeFzHq~)%P-j(#p(X^&1n`Vt!^t#yKx-Foldvgwg2>|o2H#k z!YjMtH*K}}e~TjJ>-yU?PE%@%JFT>x#%DZ^zwO5_`=KA6Skjt|p3r>)HAhLOJK5=U z(oRa{`qzoGcA6$pD@~GCJ6qcBq**eV@+WE~(PZ42nCQf5l<+!7k zq%l8fCy7<^2`vhwtV*Ga(r!mQOn4F{$z+@*ZMxFtT5qWMviLXn2NI(*IujT|H#<8Q^}GMl{?X1$ zH@)=I?XO6p{)S7oUb6F&jay)f=vU(>coyCGiw?f!45RwrM&oNt)1e9 z*GW5~`)Wun-dV@siRjOhpiJpc;RE<8|Dy4k_YQ)d+Yx;$32L_fSy}1*HQta<5dCEm z%qYcQcG*Il@mDoi%2Gz4zfOX6<@;SP#g*B(eP{GQGI2LD<>LOG@sEEs{;l}t_-pa+ z#lIil8-FK$U2<*m`s5AC8{A+T>P2%aQxfx*W*8q|0F&Vza=>q?@Khuli?W0o`aH^Je0BXlOyX{>mBc@GlfyKRRwvP@HEbu-$#mXw_jk{qig?k< zTf-=w&O7dvGo#c^K6fg~6W5u^J2N9a%cDGU`@R-!9kundW%H=(jvY(4x%+t3bMeln zMBjcp&^#@QsAKm`-pzZw+p^i0YP^r!v0MJEb-}Qkr?lH1MtOHC>h}$zwwt&B1n_VW z_dNtD)s51byq&kJCZh!^_qDWy#ON7QRTX=z3K)Cm+0ow!&~rV?Vi$#0X53_+ZqB|4 z$|FEZN^IuQ(!Sg6CbQqrtEm6XI4*FTZyu!{w|Sb!YmE&tmK{cj!A!GI-yJ@?;;Hxht zajO6s?hSo+tzip%wRR5Mc|7bS)9G~H2Khj6*$mHGd57-6N&pO&XDHnPyTi@^s)L+X zcEy!y)vz@P*mAAxFNLRC#i>}f4P~t@-%dyEfO`SVpH_CIuPVDLRLR4h-VDNB;LIdiM@5lHkn)FG*K(hhys%X8syC8JPu3Dpdoh^jML}p1>NZWTKs?i(!p}$FoK&um;V} z%Nh&mwfHpj+L1={HNCb_BT)vmXQNY6o<|pIqy;J$RMYWkv~>86x%t)fRk*A*acTt( zAR;2=*{^!RH)%9kzWkYxuw}{tHgo$vN4@SRAv$8Ucdu^nUuo)N`Vfmi;W|gRDvpNJ z3YosE1_b)72837{SWs(H&8D7X0cH*IC!5KxHngWpJ*YKP;JVI6D6<}2q3RYt+?i>pKBL`ALv5`vN@v=IPL*;- z9;BRu5Dd7pn%mBJ=hpT4t58_InOpa+OP6zXmAdAi>QUxi{d&OWMTUAVpnn{^k_V(@ zAJ-TCEfoZ)$hL=oepTh(u5zQ6#IYm;iUmb!Ko-~&gzbjW7kKVK9eWFMkim319JKG$ zIS3@gV@PP=<&%rV%R3$sUJfq`FJ7`ft(KsP?A#(H>%ghwQ8I%IXS%**IA zaSBbREHAC3M9*7E`D;jn3F~?wpUo&F*_?}ls)5H1Lm4;}2SwN5w$BJ_{X1$}tuBViG>8t`CEz=Ir^mB-EMCo&uG2y? z*hI1@m%Yo^BA;_itW9&BtjdG+DjO2z#fL{p!6BJ{w5MV(x)_{%oTDiFoM54&GS&s@ z#QAa^>Xsv^EXTny{!Z$aBdIJ$u1;3UAxYfIne11u zR3CYi58M5GDP0>*V86QR9v@9hVJwyD^tjGu&P>hKUTRH-8yH+T!B(#V-Mtw|(}hij zUGvY?-cU*=?|6>@(#j*CxRga0yBw#6=G2bRv@fpo5G}1uF6SYn(E$5r#3JsY2v=n~ zXj2d9zR%88>J%qiI(P0fdiG^em5L`}f_#d{hZ~QfFVBuNo=u4f+|kA}*oJ4bzh1A> zAo0i&H}MtytxGR2HKi}pqDyavDZ=ZcHgH z!kt|Z2#^GioCXkwPaOywLJUwx49veI4(q=sj?+R{;+R&-8ft3X%0ONt)nc#@Tf^>y z5qb_6IJ?30WF%qJaJCqW@dbWy7^Wws6O5h3PJ0y`_g-gIVu^oy}#||AC!}cQV&_Z@FIi2>8o_hNWvK#@8(_^{WE2Itzku{gA3QWvD zjDy?alH0gJ!v)rZh!w>=5f{Q(dO8@}HwNq?D7M5z1E=ZhCL9+x9GMVRO2UvSI93VF zKNg3M>G9aNOLdgvv131TUZ>-~}MSSP2BT=3oVwxHYVWTabt_2e!Y* zpg#FP?;R$#r#sM_`*aQR1$hjW0FV8FS^{IIGN2hF+ah%9H{yy=ryMFc@n{Cm3^Q0) zP!=jUT*m>!3%Yqap@MlvC1p&rLD_qS$A9I)!)zlQWL)(~2M_z|Rnj5jYtzu4Re5B* zU3;{U3W%~J!U4R5sR;--FHiRi4j#sMX%Q;mZ#&iCVRn4*Fu=osW@X$dnbyBN=J7z@ z50R(ih7LYxWaVTtPc(Em!MMQ;rR*`z<#I$zNRTC6VBBy{Ic&K5Q5rVnCmuE|Er$&g z8gESI6UDIMtT1dS#v6$PByg1%<9NFA0xv+%oRU8b?B?BT)KEJ3e#~%=9||PyER_i_ z`R2^zESzai38Dht`)5X@^!}I2`=1#1{?Dm-|Fn8C=YQq=&i}GU;{4B!JO3Ahf#4iH zU37N$%lSX2bpCowq8Fj%7ivE_o%WBOdix7< zn|IAmgWG1WlvfrcDnAw1+;dxS&HW8TtSIJ*xDdwD)4|xjF<=)#vEU+m_z0YgVXzvw z5|CqVv6!Qhovb41W;Yjd7LMN5#NZ-;S`dit=>jqP@lyt(NttTiKgk}HL}C@w1`?Pa z9`l%#1>pADMBToz_D??b+<#j#e+YpA&Fqx`fMvi{S-^FIKCKaYUg3kviF%a)AEir3 zxr)tt4MC)Z;W%&n5ad&g9TbtzrkaCufYuvTQ zEFX8r&GJ!9@f4q-Airi64dAWUnWGU}8O&t=s1i^{O^mT8GxaNLHgjb`nFC{#fjtNE zUo0uW0_NWK@~8&HpVl%XG}jY*;50F>j@b`cJf&C5r6NCaXEnc_T_Zq88ssVrG)$6d zpyo0lDb}?iX-!Vn!rJ64en8S{z9yI+SF<&gTUK~^X-;)#Cd}3lC-qG+!7^w}*a(MA zzP1!s)!Zs|5e)ax6G(z(1^{;gbqsI z%Sf-VOK`iuQf-+m>e~bcu^LRb8qdg%N-pQ`DUQ2HvxzsXB*Bd0Wqn0Y9g}mjT?Ea;nHbqqE&3S}-WnKt( zkth~hl(>$h=Ql_^5!Bp-_!xF3iqpd|2)T5m%%BT^y{#OT0V#7q0l z6jN$ImKi@i=6uMlyDQG#syBSvQ34q;8ror2a0;8C`c!Slz+RsBgOQ`ZATGrBZQ4uo zU6cB*=hl=;+U@-SExX6LBL>0S)K1i2SQcju=#$Fgkp{1xk}ST#P=1QCc({(|d1P_n z4qdsp4jn!H4xN^AIdV!JnhJbBb(|;>C78^arOo!LpZSz{OIn+YAX4};t$eBysc$z$ zY7nYsPFo^1C`D?jDN;)cflAOWHIW)fq?YFVAX11f?6)GHyou!DEd+|@_Wq}>GgOMG zz0v62t3<6AL~S6cUqvB$p!wFb@&Pw>Q5#6qj@B`z_LdEoP_8Cw6OKrZ+y4<5K8PBn zq+$5!N)Pd9FlliHREZsoEa+Tup>;XMW%73E7LYJ>Si?M8zjh}wT? z=)}o@?a6rvnZ3$LI4PJXB+DfUCq%-@iiGcE1xztvh*oSUT8%44ld{@D9YPblK;@;P z`S__GgXFF^O5{IY1u50d|4z3 z5#D6dQbSS#f~+5fy2)NIK*5%AbK99CZzdC& ztbE1}K6CJ?WQfm0!Nztv0=UP{{|W|+TCj1oqWg- zBFZ5@*NYkQw<~`bgz^7@v2*qo!_egwf}my0;wKcTPa`YS%pI*`p&oId|BhXZkT5sa{vw!Q zK_qxdI1MBm9s}YLAfbCYNH{Xq{v$zxSInn@grj3XECLA&Qo_3ervb$5?Mt76361U%G(0(GjMSXU}*_j0Vv^^WRFWYb5XQndtVA*NvzHSfO*{4*w z4V(S?=VTg)Z7|)bz~+A728!gi-S7EYvyWWGhqj!U^~%|=s9nXz%Nv(x*Q>}VEPEwx zfH&=~zIsj=(rPk>m0FSe&#F-CgxiiVp4|BF_$NJIMq}}B`X>PDEcDJ2Uqy-SPE!hZ zG66)t+*?$;8)UbrSMIp67K@gdq{wa+uu=cSDVD$v@QNZ=qcwbEn}9fZisgYie^5u9 zN7<)*v&$g2d$$4lIBb>crt-piPn)A?)i=l2=%*E#oIdpSZok#@b{q&YKc00kFPY z@g)`i`zZUkK*+%uSXQLy6S~iS&C|<$@;6oNFX@&pYI(DieMl`nqkr^qOFC?y#dayj z{a2f@I807o>`7IS=sJBrN!tS8T-Gt;@4 z$O!j0$C^wyGd&P1e6~8dhIs(p&&ESWqpOpPi79dWA{Od!_bj4^+)WRc>TdNg(_KD1 zLwCL5p{a)Or%c{X0|jl(X=l)w9>7&!aV=#Xfns z`otbu7mlpt4RKkyAy*a;3n%Vayv^;IS&~GlxW^^udfX?|67suE6Yp zel0w&MogCZWt!Editezdz_>QKU6;$&@40&Y>=jq;y=LEmx!FBetxFH0*i<#TOt#7{ zG=B#g55_auKZvDehL;YhlU%E!+@Ciak&4haGq_DAdK$QotQ;8H`uFx zWj;i>#&F24F^>)o$#ZyQd}z4wI1Gpx$Dz=4P-xP{ZMw)F5`*gYaPj=^=JTV)^9QTv zuzmhGIj5?V~}*fUe+R?gU{coxc;Y}qrf&aI%~s(ctq8VB(ZFZbpE#MHX< zK)CF$OXtERPVb%Jk{JE{;c|lBhD&;K@w()e;x6cVYuV6lEqkkAR_czTqYeWvizUaOjnEPCilP>)bOpbJEZRlzc2SC z$ZO$qX4N!sqpAc8gJa3{NRw#gwkv1Q(rm5(4;YL!BLj-mg~ z=B!|#UZ6wbtV!aRYSKm8E=is~p>s9MOoaZU+0-}d2|==!S$IZh3;0&_3dq1a_ZQYCy=AQ@JsTlw%X#V#o-TWLm{~BdxpH&0)c$i}JbNNfU zW$+2^^Ca|1WpPe$V*Bro8-xte0c%1_$n)g6elVDQ+QSYjYbYe{F(4o(n2mRG$HfIJ z@Klr;+tXEcNZ(MGVgoi>w4j0h466DxhKO^RK)x~;s zo87Y=c8XnRmp0{6-D(a+(LOgqo|ze)i?^del%Pu{>Mn*@o#H$H0Xw`&AnXe^=*&C* z!>&70erWHM_r*IE>4qA7$=E$^m%GdwK`iK>ZM$YP4|KFp(gQvQ1S6QT>Oa_ud$l2d zQ)E*44WRw(hT>Vb8gSiQYRbC z#oI>8G0~0>6X55WQ|g=176mIxx7Hs(!FARtBtvi0oiig%@luMP)){D-paP#UMLV?K z5v@s?3AK|SSBrAdC@*Hn)NJ*wblaCsTuQxp@{}lsL&v`0xxOHZp!#8gt~x~0oh5ua zgHQQ!v=}SwK!<4p_Ff$^UY3@bP{(5)VQ zs7VDTiUK_qn5TxW`off=!I0*CzkMGVdrI80>Uo)mmlKVb6JnN$r^QS7=|3&jL?W&O zE8pv+pz@PZK+@LM+_c9K@O!R_Ox2EQ!v9g_*JB#uU}a@^Ob83T0aei&^JojHJ+Cde zBImoVE%ZbojzU}DYtR=AwQt2wbwUX*#;J>b&NKy$8)1wRX0uUO4xOKUQYvcL5>4Rx z>~??aDWY9dV(BJDlB@{q$rbfuV<1$XDl-FMLJDsrv_}2!w~`)F9rdhNREJ>mS|a(rruFm^hA5W5~EL zGH~D5-jp$lPiYo}-=b|kpw#klB!VD->3I&Nr=2X>@e-s}(B|x3LY6{9prb(>=+XA% zijSsl5{8bwVw*e2Z?au>Nfm&-i0mwWSVfwnT2X6X5_zT|D=1P&ti_>pQGlU4-n?65 z+xLw%-(ujbHrN=YQFFo*DJmeY>pu2tB2Or^CX!90LTYF;($=5a+)_0bM4G?bY1yt? zQ6@)ShsB^|3WZo!W+<5OC{XHRg#vP`4A9AD70So!_0Nw0E-{|)69_Q-aU^hLZBQ)r zu+y?B=w@~OuO^A&lTU^)cqEn29M~qp;-j(v>jDf2w3)S?-ozq#{2Z`mx<05IDIa-d zgRnK6zw;Z#W$uR8&dnlSiYTRpKU95g0sN2;p}pi_h=nTp?)()Yn1bUJD!T1;^Fq|9 z=mZ^_@AGr&*sSSBjW}%ZZ5}J=)JhX6%tHl`YIaIIxh<#v@MU>}r( zv8&&N4?V@5$9v`u8G__myW4M|iv_kn+>878mhc2@M*=+jGdpK?&r{o8*;CY+`IS$J zm#{n>2Oiy;h?%HVUXxtG7{lvcxoy-Ye{W`lecv{Ouh}TdbTb9ou%>QuOeykPXd5c~5PkOx(jZ4ZwTVC(*Bc%C-NrCoS1CE!lMm3QTK+aqlSKmV@?; zDI5qK6Z?bT-xq|I-idu>YHi9L?QWBrRU^`cU3RLn@GtrE2Js{qm5QRJ9j?2JRPxTQ z?T$!dZyP|^a_-z`+(#0>vK=}kg5lwzBi7c^)m?{B7O-F!;>!%n!~E_dvT zhr8RPne;mYL$tQI%nVN}AAmtA^<*cC3dQFhW_XzFtT2f-Q5L@MPrKHi3EwsdRLVYE z8w;~1ZBV|LD9k8?!4;n-Mz48EJt2BcOer1C^qj{voJ6Nv!=A{jezV}f_)yG(gSqF~ z@PtuZG=fPc+>7L!Ps+V2A6@1jrT!5w=?9(>*xZcimY-HK3-F>sTX}D@EIaB;c9dP5 zsWNYE9nI$77UvvPU+L688CRzTTVruOAbcG zDO&5Jh#kmkIeubFI^LEO2%E!zSy&) zwtja1(oLw2h4#m0sm2NS$4<4=9J>jx*EZSzb%^rNB$yaCWhTZCklX7gUI=k5Hc@Vn zebWfnq-u7dJ+kxYR<>y{c5FS5O4`r!(GGP-48@SUebWLvWm}gWi_(gnvZbMm_)VVT zOuW)9D|X7hGp?(DEsHGmznlFEDqK#m*#PwfZ>^b*bEpHrWJT6c^HG=QU9B*c(4akvqyB+bCiwBpbYux z`GKlLn*U^Z7|ThKSis!qh6EuLJd1O-$+OX%o)_;nV-~vda44tPUBck4E4`PRZHw{= zjc@t!9 zr_*j0>p~L)>ZF@&ocP-PHy@66Y|fW#_wR~$VpP6~wiCg?DqS9a)DPR>aF9Bq>O&szVMcHK;LK|U&cd& zA98jy>2S@|CW_8h?EIB#b=SUAbu)C|HtMc$ghcZ03cVleHH@K;1jNE`-5XVzTcAV= z>Fco@Glv0awGYq+*PE;oO3wp?5Kl&sW)zn4V~DY=pS-8QJOH+J#Uf)8f+(<55ing@ zK9HgCu(ShhwF_vo@C00u1|&mMhtkO8Ew>Wt-M0;EiN(L{|~{9s*Ft<$mVzGL7JN2`~WVrINl<0EY< zEcRsZ@39*ER(CXXr}4d9jS9gX98)6NdU#Zbae}@+6 zpB+sRm*QtIink9V_yi>+od}-bTJFs^u4rR`xVHvj2pF#s2)!na1VP+>D-;)GX%v!z zFM?O%tZn_LH@Y2%ZpZFbrJ0Rkd70$Mz3%nMVouG4eP#Vi>^^N8a(S?k3)T1PMc4NZ zt1l1r{d@zys341D))nLH9=sKhERtVy3?hNs3*ag-!tMPq#R{lvtHalniD`7|UePL< zmhB-Q1l-@Fslt$tcCNouHd;)lwY~|ninVZSB+oaiDDbSYDGxq}BkXet@R}J9cK`MX zJgahbczr}pl?TmyDfA{+QHF>iBf$MI(DcyawyT)*r{gL4FE#2yGD4$<5 zO2bEdf$M*4=~X-qpXO<};+?PgYO22aG58MHXVs6{Jz}LC>;HoZI=;paD(K|V{h)%r z{0A5G{09egTNYRfsWz!8Y(@V+A+kZch} zfEAq??b8T#U4-u|#UHJXuCpO44`Ph0KhG(aaqNx!YK7(0DD7mi~ffgk{8xK!)j8;7+NquU@AG8 zR#j)F`TB2v>ef$u_3FF-;hWLbda!!^Lswq^nYaGUm%jMPJy-A5z+d&_)zxb$zJ2TU zA9~lZxzE4xzT(^0-uK~;f8?XT|F?e>g>OL>z5=l$@)H=wGOfe_3Rh0CMIcdP2!mV0 zaWW5%-KUo6tT3TcQ~I77*v^s~WX1uEjZ-uUpt&kMN1q6wX4D)nNd9uflqkXYKl171 zfKi-e)!5C*-EPQTXB~Gp+s#q^EtX(i+{iRfys;HZ4Dp`UK5aqN%xzfY@U*fAjE5LS zF}N}w2rD2ot|~JI6lMtayLLZ)2!EO;u)F}Vj$-9Bw#A+^;AhB6XYp3} zK!YKTF!{AebCL5x4P|p95G{}H1PK~iflLwo3EPW@Q>hr)*+E<`>aW9v0*->5VtpG_ z5-0T^&Y!K(f2~p>JYO=LsvEQ$pQJ9mBxLfQl6h z=4?7e;)v&NvDZ4KL30fZX8o6}!e0qn02^0qD?tm|pa4kI?As=wY@k10>wSrzfhw_^GmSVmf!NHBT zxp01%gBMW6*sZ@x&>Am;zc8b2(HZIy2NIQx`sX!<5kW;r?@2~u_otte`X&G*+wvOP zjIz7cTO_v<|2)!y|JdF61r<)h0ObrGNl0OETifxR{@ZJ5FjbUVgfZe!I)`ROdByv9 zbXJiHv^rVCoJf%XVyNeKmp@g+W*y3!T4zTLPaqe>2E#U8ps5R=HQ^hZvlJOH6kFEd ztc^0kJ((aQ1w_sR-rQ z#k9*0m^n?A!&iaWDSoli_cY=Rl~W?X!1ugkrW*)za)vFr6Q zM(Jit3>?K4F)K_G+zwaS?uBZx;JT&#BD>DTdAG0?As_R1#Wfmr${lbAE$8lIVk#X_ zSDq(<1xdZWFjs|7WRm_~Y-f3ENQGV4VK}-*bfYA&y3NNF>2)x=7MhiI+zP7(7S+F_}~5^!h`8@wpXO=NiJ_IwHd*UjO0!e{o%`;W0kw zPy0g2bU9Xc2{G904~0UFy9JYfw7ef;k?>4>6Kd@NmMi9(UrJ^wqpM>R;d zdz+glV=1t_Yox&TpVQD+t-vQOB{j(kz$r7?KdYZqT(}T$2u{W3B8#@>NGw+|%R>Sr z3`((1qrVRtSP$WVJcS8bfeFB#92LviAg|&kL|Ai7u$^M-wU^i@9B&L1IM~emyOKC% z@#>5TRwqk3kdj+kn!D6eB?-~sk$okDAzs(U|qBgP0_DZ%!F^yKr zni@?pNc5>oL81Xq{5F`KT zPP-7lK6Y2m_hK-~ zp$TJgu5DM9c1>qew~AB;*e287?3nx`YKR%EDjkAtYH9vK>d!vnmx>V=(gEu|UK2A7)ZlxRv~Q`KCnh;6E9 zsTgYX|D#uQa49>Lidi)i$>{D2Cd?`dKCLF;lG|a&IhUvC_QEN;<>ptfQuq3As);1- zRKsR+{I<#@)0X5tGJDCf%D_ltD>4f@n_^&GO9#S!Vrk8cOPR`VNDD2oz;>;3w!S~C z?5Jyr%EP18Q{B7RGy3W6U3_mlo(Oh@P=*;MAb5qeu?-X8G-L0!v+3i!P3{g%pQh7E z_y?D#nqeJv=SUrjo&~WwxIuV;Ffub3q)WioDug>`SwxbGh`C>z;3dl@X#W=K(AzX(!G)Txkz!tg^1_k_ttv}Gc#v|)(UUcMUMjjZn@bG zGjAl73q$zup>R=^sj`5uXS02(n{AO#BN~~%F9nzS#HmT?9Q>96KWy&oPp;Cm6)CcW zj-`Y&N#QjR=2Zmpp$^y$iyv4;w1Jgk`6(iy)&q$*#FVg!MsnUAkRB^q!Vf@pYW3UJ zYBVJ+QUo?O&TpFSQ$LJ5^7*;h$W6I#+i#mUwo9M;yKryAyVs#2qwII2lZJI|mPh?B zrPTt4HKq9w45kJFWf*}v6RgPna+qj4-Wbcx&UIWV#8CDT76x?41>tL(232|@IqnoA zwM|lz$K^aMtfb)|%FbpRU+%hv(QSjlH6cpJ zeuXbyh*PMCNS|CljrzY}Y%$^PvR^fSt$4%zo_}uOx!3E_{Ot1f+SdL}Z$tDne?zRE zDGwplRW8L!cd`$tQXhwvnS`q*q?l9(_@J4HLoFTj1v-l}*1Ou{r zihTF;bl0^!=HDPBg*c2KafuE7DHJ(g7BupzoRd86zb|c!whK4MV_cw|fJ`nGw$H{m z`V3-Htk7O?H);$LDoJnasKwtcMN?QU6gpAHe%kkJ|YFmI0`-wk&8T zY!$okX4d~AW+hi(!JYCwcnj{7UrS18c$BBUWxa1`@mtpa`^bxdGMLD13}!jRPSGfS zc%EdxD~lQRzZ<6-J(OXrx=Sus+IDNf2hV<476Dx)FH!bl8O_L&(v0Lshry}D6Xh8w zLsqs0mQ+YjZl%gn4#TFHGV^WjjiwLYu8)wzwiUpdYI$c^<|Hpagii7IaAv#WSi%?X zM|NfJ(1*TOt^XtQgHf49Xt2^DdTlQ>MHP|E>jTkgIatWs{k09HmaYOK>O>_-ec4Dt z%M0m)KxUNpZM7NUZl$}Kn@lZBWRxfxMJfHXcIQ>rMjjK{*m3J9+D$|%=M{x^`$*L9 zc2eeIc6W=x?cu0&Pl6%?Q|!|!k4q!TWE!jqd4y~TA0zC$9NT`kU5~{FtaI3AGeT1Ko%R5+| zoa5uN1T4$Atf)C+@wi-j`o@9H{NQaqojSqSIowSgw`s|mymymaCrE$?EH6SX6N(Vu zDz=M6oT7e&jZ!Eh4w#BPAk*kWaj+T_Ty~?jfTVYPbC!utxq&*Z51wwGh@kB9PeE7p z11zaBFrB?qc*@?Tzgow@BFRu9+?^R|F51_tH=GTuZPcyu)y&xFEEpznvUhuYgVXGQ zF6BZQ%19Z_-ec7xOY~gv0nhE+Fb){CloO0iQGkI(0}5Hd^Gw*`cWb32p?$_STDiwi z1IN%qIUMAZ!zrrpy<$gs-<47CrqPmZqotekDYf4kWt<7|UbWA@<4ZU1$lKdsA;wLU zY+N6zS_*m%)+HRC%~9*w`wU)L#bY+|8gPbqlHI7MR&Y3l^QTm_W148AD*AyevVMf1 zxqrsC^&WM~nl)tD(3IeT#$f?7-I;s|l0BO8wPaN-xgH7Y+56Qh5{U9*BFP*@w#5j7 z+v<-zL=d&mHu_}}Wanle;`+S)KPPQP)0Lr6Uwjbe`XFU`(>8FQuK$s1)*p+mT3>AL z6Ys7P!O<{%)SaPgBlbqWdX&P0e0l79-~FfW{K;pJ?SAD|&CP+9`;SnT2k)6*)$ZuS8q3$+YLfaa$+ zny^K_e1YDwl~;p&us(iO1GM$g&uhpE%R#fzq?)%$8jUQ8rqRnKHt5uXSV$;>Gn8Oo zqT1d^K4i|JmMlZ`=ElzmmI~Kq?L4s+F3MBU*$Ae^XYn?qr|0jL{4EvibsqE!++?5h z-&4zzt!6{K-b_#?jw3nI;w-^9b}YVteDFQoq&r=AE5i1YU6L)Ey_HgD8ca^M3(Ng| zL>~Q_`f$8Lf9PCu{e)6mxnmLfQI|H_emOQm7bYsTfD-vd0Lon(0>?66*0H&AHOdMo zRjGyUdGF!A87Q=-(xPmz#PChoT?Ez#D0J?Gm-@U5iO`hvMu7PL`RuQ!(^1Qi;0n+R zd7vBl+6C`>6HZu98R`7vWzYpF1eZ}A*;Xccla?sAN2#{3=ecWnx$m#WawgLvW)@B{ z0zG`k10z~FCGi$NdBvv~B!JlkMBQzigJi>>VQdm1{m;iXDbtc3?_N{M-Bm31Klc3~ zQqkU5YJNtMi=m*@C`yU}SQ2S6K}n8E;Zj2-f!w*7`eN0SY zmc$$M2{lZ895CR8OKu79;z;tA@-oAaHh=*M%`SL(Q4mg)I9^cDUDX^PgWUfBNCy0< zb#|m+HtcCMAyHht5-AHYut)9wPeX)sI-%=RHs!LK@9;4gt47gUJ_14dK>)N&O)El& z^hQ6XOxZ}}*yrbFBlmx}8@|AAkx+Y?>$LBGse({t|R4-{pBe*ZfYlN%Y`Y= z3bPz5uMxtGn=&Y*kgLx$CM%}4ALY{?Jkb_^+QU^bOj=Fhv@T$W|0!<1@7sTjH{bU_Ja(J!`}QB>&G$VJkKN|`zWv8|^L-D*W4!sk2jVm~ z-+y`K#4#icD%aIdCVjWuu~EdiX_0wrKCD+BsjI<@JY+np;HF7myV=jw+%!kBoPzJg zFpHx}zs6xsY-11{9IbRH1`{}K9InJr0aqTS!g6vu=Z4UNR1C9w7GHe2k1w9X5An7& z3<((YoyA?3Ev>o3fy}vMqcftZx+Aqg~pM$6zIW2@O$p$!N%o~Kp@FPlef!Lmi_-6Ctmt#nIt$}&_ZnJJ%` zFG|A5o9agz&1>+djjj}yD_LT){|h!W2928LBrvR*Ee&m%iS;9Jem`&6W7uYa6ktn8 z(|B3-1q3^J{_{$v@~u?sm;cvJs2as5fu3&cj) z%eej*C0JElUwfjrLm7p=buaeOIx_`xOojL$%E@B=oqyX{zf{!mv3~2`smA(M)uLkk z=_$tgg+T2Q5~5K5R_2Yxj_W>W{gmtTwRc zCmT{obD(P)<{F%JO%=x{MdAz^6(HpfB1klQW~kAVVq;peMi{JhQg7-i#r49@F{-NFmID- zSZ#ILUGpoD#4U3-=(jlb)!m@pyC=BY3=82fSa-;t4&D9rr&P>FSVV9Ms_H!5#>{aA zq)iYTY&MPTL4tp5LRk?!fb}4OgUVfcn%Vkb>21QHs~B{W-J=ktf`Syco~VP3N%;PB zluQeMr|oL}7+3==|9@SYw?jKM=0A_6ZfeG2;Goz}e(3kVesW&NOK^L6lzrB#Zbmh; zD&-#MZnM$>i*gUwa^6u>InTwl`Bzj`P{v-Klgj6h`X6tn?YeQPvP}#4iaq~jIrJ}& zgB5{e=VB$bJ2Fw-Ez)yEHl?(E9@mFz^k$jHlT!6I`HhNB-M$Zvb=;gV<+zgDW&=o# zK7t{{7-&KwN4>5B45@4xFsfljqc8?olaSZoR#8y-qEgosDl9>w)5PnRK<}{xfYutu zL0^V}SBwI-G=_!()(249Ypap0ga)>1gFGHBj^j%C7rL^er@@eH0!iWm=1xxO((j=d zbEzHWD1P$?gb1&*0A!`|K6Ji(985R1nz6Jx!XLEdk@$?OOLI*g4eQNXMjSkCn<|Gb z`1mVg9*xxe?U!`hG|k#;O(rjNMkBZ}`=-ge5x{7M&Jt#X^C+0SdiP3`cfhDxV^AQm zrj;^L2Q2Bk2h?%5#@%oezonVH-JU+#-D_wmYD#bfR+)p+|6UrGDM-wH#r`qlZ_O4m zNma>~e2#1E2sZC?FsKE)o{?>3F+Ce8725MhyT)+z(gzEeF&7d^zti5!gO`TkZWm+; zFFtETSB$h+_Z@Ik7drTrF|$Qrpf;-ix+X}-MgdZ0$Y!*pO?v{1 z1*hh*6kxRiSUiPvC0N}676VP51>5d{GFiO23DKZ}h*ceYF@hye1X0*|z(Yg`q=qQ@ zC05PPZ%ncvt-AvDuL?)GNI%PO9d!U|ej&WmZXLP2}qdESa4LgMah z)@nqMlU7**^V*9#YZ_s!PMEbKGc+l2dXKeG;Iz)8l1)=>-b_^J&ZGXj7G=la>x1JO z9Z{1it2)rG5qNG~Qyg!~4mZq59bV8?1h4wP)`Tzhe1fRe>}*SN_kih{a`;hjIsL96 zG!(>lajs6XA9WR3#d zk$aQ0ySDtM(qh}QJFbP7xu2cmx5#;TRjXQQh$vJ&XP?Y0(KO+zIM?Z!^q8b-&f|uhF^y4qsZg06r ziYSlxOxe@Ig$WSrXINP;3%5YIS%dcESh|PgEB1Xf2wb+QE?( zC(M$Ad+2}|sKnX}uwa}E8{vqoPM(Lc{+!6In`iqVz_W0@>g^UhQ~Z+e7`zU7GoIV- zZnm*f=0sI4Cx_i@$3T0=u4u2p=AXN zWMxBv1?Fj}VgVZROwedZ4x54rEw}$Y4U`RrMD5ib!s($qN^69b=V^tbNCGR<3YRc* zy%lXiLfZwWnFP&2jFvk9t!`^J{4++YbBt8eby7Xulj@u?QW3 z^Kde?1ec(<4#eyUr1WGCoWU1k*d%B1fD-?2CNVnB4of$1b_K>+gW>~mL-Q;F?nCbf zWYh+iXr34H6Ex2Y{FCQZPoC}esj&NVjFG#m)I3Wxgq>H>Y{^9dYCQ}@fM?j7)CZV(B6gy;^YR4_L~Me+-&ER zG?3pT+dM1Z*=9Rrg3GOdqDPyJoRS9Wj%%PsBV!tfqnr*{TSf30?pd2a{m+T|Kk11d z49j5?jzW8!py1KP7Tjk|FQUe|#TGoW*n+cS?_&4B6DT_$eW)Bik?2UN{NCHHT)*bz z5VorF{LGs;s0K^V&z3RO6`6Q7uf$j`Oi-fwIIzF5C8fbKO$ zsV^F!n+^9~<3}6lUaWeyHjcC5^)|)?$V@mIk#3hFb5`k!Sw}1WUZ~t&1NP+QZfulW z*g``MjBhZ|?>_||wfY}T9=+yQ8)SGSJbGmBwAtid;*9+7P;-S$VNM43oKB$&5oCCO$?naWWTG@Bc7d z5tKAVJ<++)5=cLEHqReBZbRo@7&`a7QZ2@sijM@h?wm7TEwu8VAGcYiNJo9Tt=ZJT zHtnD(Mx>#sNwWd-HN|c+g}s9?2)%N<%|x%(RBykG6%Y5I;3xtziH z)}lk$_t(VJW-m@5gne`UjB9xjRRH^;X0@jjzR=UtM#qG^$3gjLqE+CIC zT1dJERXrqqhcz1}1B#IJJDNZz0qCR-G*h>=K`Qb6lgwEku+qh>gGES-DUl+fXmNr5 z+nd0p0XPkZYp$-e=)q=a_>?34_n&^Gf6kg7L+j!pzwILxGeF8yh8%`4KC?f=Q<|eV zWKEAj{)r~e7hnkC{O%g$1ydkI&G{ec`gu9v!4qXI1V&EgpIfU1!E zZ*tlg-}?w5uVMUjAiw1iKyKrT)8>`^wei=f^UD3n2{K$x%qus}2lw?&9$!FH-#>2O zP}9xnS`QSijM{>E4Mke*w0{)#sChBmZ;C0;PLg1^UDdUNL&Rrc?r z?Bnv<^Y+=14Gn%K`-JY5Fx1W?iP-WwE1G}E&bj6$j$$^EI7BcO9K`Ilx;B|pVA?p> zX3%M`Wv8S(i^p#k>civQp32@Zi+!E~4kS0{J30#r;)w4Ko~ROAm~ zSxohB&~|GlsUC-yu$v*!@FvuYP}c~FvYs|pG^{LZVudX)o&z4(pXAw|Y=D$2rD$p>9%IfOX%>E{*pen1Pe%Opv zQ9o?PHi&^U*gHb|V?er?eKvp#NTtv8q%B52%lBb*_IK(Ry|2WkLm?2YQ5n@dUg~czWWfJ zci#$HKOtPMl^G=hFMq(#Nv^WiibrV_Ki_sURY^kBJPnpu)WiA3SiYf1DA z7#AId;Jn0VKqDvvangQul$8qbXiNRKVbP=3fAX8@HMyrC)Lo%%lHobF!*96*V(;wB z(h7>1RGLs!Moq`rgI*rajt(1Oh=x?*qeB?L^8fPAl2*h5IfJ1x4^j3HMe7ECVHZpT zSHpT|}AD=X3py=SJT_Pl>y~Od0pixYGIc_NYeR9+8mSb+Bl*yv201 zFJKlCOtWWfK%cMuTB$4dhYMM`xS#!_=gusgMD`}XR`AK@>NxVcia{6R&>|7nq=JM@ zTSvj7K)*|J;vl29ZX;Pke}0&@zSEsQKTO-|R;JO;xMq2wZvYT=7k6k*k=SDVOTf26Nfgo;qD{_y8ybDpplxXy^CL~CUC&C{QC zXb~GOCDT+5_I%Ybi1C&0a7$TBYr8}9kj;h^C|klK{I3s#t;ja9DZoS~ZnKBl>>P9! zr@^3lXd$!ep@t8@Ui81b`-H1y#Eg8H9>#}qvd&mnC{81!f_Lp<^ zjX2JICvx}eqOdkc;znC7-T_a6r$6EBSd^G~w?M zC3#5?f1OrFWCsjT11ru`HUx6zS4UtG^qdXb&W$*^3d|pr4{X~`%b(GH=axp9ld7eZ z`?mJLViEXeJGTuvKoInC$Vt}%h}%^|sl=;A-TmgNwPmzSYk;*TZhxUJxb9-Wkm2lD==-0J+vQME7six9l#(1THZ$j)~;=izOt9QkD!@|YQ6s;=u5{up*g)p})x;2zARChFyhQD~=I?dr-oR!Wpwnup&Pthi}2t2!83adMuPGVvdK z;QCBKpkQ%H`&QicPCh|V*ozGoC*_l&`t#nc>>j9;Q$CUsrBMwmOmnCqNZNY7pRj0l zSp9356y+HC4rBDkI;@;2p<>*mpm4O2M+9w@Y0&?pwB;9#&3|}{B;I%du8idg#Nm)b zIW9V0u|-FCm+5#WRGW@2XJwI_i#c9#AX^E!DKOYI1eL|N{-a~ZQpTSCD~`#_(S!Q9 z?T=U-$k6Cxt97Y=uVU@;m_?N?k69cr9qoAo5)A?y?&YwIDjcP?s zjbr^@pO}d;nu;Su)3lJ1ikr1ni)hL;Z7rOnziD?M@qlmWmctEZd09RJJiTvlUYP z^*lA3NyinC@K)1a!v@+47z${gNbWYk*2Pp+1NgE4`}pnY~tBWop^oF5iRzCm0h7WB@zNP@!b1?!K!jcJmFqLNzTRs!VauK-9`=gyO=Vm7?%6?@|T z1eIDNBJc^m8S1TOB+pFl`vUbsb{=qdZ@BZJ-B-)vNBy_QjZIzS!VL{r0QgZFGDl3E zi_>|qs|G)ZfodG@(PWkg7RHx5JDMn>z+go@dbzqA%k{x6ard30|LM;Q# zOTwyO5YKMw;($C^uMdfSuJUgVT6d+KZIP>7l$LMli4;Mbj$%J!cxOYNhlo-b}7rX^gbj+Dp z#Wxl{kKFA|)});>i{uk=9Y0l^U}5~Q+08uo(YCL=IWc~$cjNe3Cj11?zJZ_l;4Z=m z3?d{CV28n3U@#M@8c)?^=)vrv@4*;TAAQ@Y*WxmSw4^Ku8obqud$@a>zk9rY_shB~blG>3 z`njnURW%C!Qwua>F*WI8H&xIl#MKpjLPto#v4v39$AL8JX4s^SmF<8F|oIb{2ImYIZ3+1o{>{@(vB?$6o7^Bk9gf=C zTe`}M&0ZcCEs>?fcnaW+dQ{O4!`VPC$CsfV-ZT_B6NW|ToFBa|& z7atv}6=GJ)m%XQWdzXJ3stn)W?cav7xh%nKf`KSd^XwVNp&)cnM0rqfu1#*%Hw8l6 z!CF5B{@#+lV+w2_e&6f)M}2Ak;LH zhYGl?&zM@`T_avot01MQ2NRfV4Ik%pYzvqn$^+O31h(I+M02OiKBl{NjwU$rOi0Qn z=(V+S5h|zj>DF86Ts< z!9dEC<&eP%gjE?q!$;JsFrvTb1BKt|F9aD3`fd-7PvkzkM?-P(G~Zsf&C{X8N93Ml z>mm^amW!!m+x@sOy6YvQ)@BZ~*d~$Th=-r%#2BhJOckmV@>Qs`90RE=>w&bFIMw|K zh2mYC`TL~vR_yd=oD?_?sMp9{L52><?^@d!RA;09 zc`G15X9ca1O+o#6e;DQ1^1eoSq*iaY$|jJCZYdS{BuYbjwX|xypbc+$7z@=kS_9kp zL&bD>W*9cq4?#3<@}6ga30+J8JW{Qu?er9+1okm51Z(ObN#Dzdf{P0=#)KGy>v6^q z?7mcBj0q9FC$&N8z!(!Ed{5A7@7KT>Fkyo+>^?9CnE_&qNh&V^0|?^Rpt~uWJVmZu zrm&Fdpi;8A#XB*P=Y>}Yf=kw-&{^nX0xqS_f|g7IfiYK!C6|S;f+hZwMv#i&SKKOke`&u2NQO&`ou zHvNPa9h<|*mHt`GOnCWnG&Aa>5C!4pQD>Vb`VlVVVa-}U?_lr=qCCwYhV(&sO+d^+wkTp{ln5I0a8MjNVl(gArbRIOW~$M^856UOJvk){kPGQ_-zWBl z8VO%leXIAZ!+w40Sr@vtE|_AUfKF%~Ayyb!$K2%6<{Z40HR@Eo0HXEBcY_U;V1FNt z2kq>RAUOkEB#W+7^JcYdR3D{+IOW*+)PpJ9atA#EW#)7 z#%N;ZJmPd*8gppARiEo)b32V!^$EbyPYJR;Hxvk2mBYy^YXof6+*2wtk*s`I^vOMo z_C`(ll-_8#8}~--G1NBS#4X^BhHvkdE(vji7Q+qL-Fu_Z<8qK6d`y@^&Ll|MObK%% zJ(Zgds?csK2~^O$;alu%F0F+*ksWa<22T1FJhPjrKlJr-*i)^cJz$HV3q@rtzZ9}H zDyzmb{KI*hr&C4qpx-y2=KavT1x~GQqZx42DF~b}epGOV$evDn-oUyUYZhFQXkY5H ziN;GQp!)(d1qaHsOf<3%(of?nP)GdNYaL_-t<%IW!YcgY+m!ePo(T`Qu;tc)1<*6& zoQf$hoz38&Pq-&9gp3+3`UNs%I2Gl)TO6lA&V=p^eo{0+#{>pst$)qk{Ok;tjbkai z3Tq=QIXjO`!Dm@4NNaAZnRuD)7DtE6r#HYikD!fM0H;NszA(+T&J#nRYF@Lw5i9uN z2`s>C`OM-Nprdr@Gt2+f$O00{KQZ=`vk4dW$oXSqaqpTv$;=e31&^+c+R_r!&_XHmQ_~cPA$RuC)_uM<1+l0SK!CpcN1eNmUh$pc&5)DZ(>s zdCw4>DaJDaDkl_SH0Kc4S!1>z-6a=S{+)b*2|0fwRy=2;31OKrTaaGgK%b7P!!XDY zIN&u}q(%&dhYQ^$q-X>|=YR#bb-DjU9oCVzfjSI|xk9rQehHqDDc4ew%0Yp@<)D;G zp{@d`sgS|KK}o%XLf(cNuDJv<$x^4PPny_ExrBi2>fRyyOkWaND=oz{p0QT>}nZ*K)%ncMC6agks z)A{5@c}J4MbQqxLg|y;HUf@{$L6sHp5!+0B+8}a$rHspJa3lK4*Xiv-Ci{= z5brQ1WMW->fOhc=YVi(-bXS26^q37XrBg)-P080LEkD#_sWkMzJeW?rc(S_SHzw71 z7@IwX&^%_(`psUc!#L;?Q(HbfMDunB7Z#cjhelC1KlmcAycA(5VxL=(zYCR^xO&~dWt5JCu`xB2eVMC(6BJRqLq@#MTdp%2NozV>i6*`oRl0Jd2+N2-cd6Lp zLa_@=Y*%UtvmwAmZ@b4ylEfn?nOm~XH>Ta~3Q0m>fu8Mjyd)Lfpk8&F^H}r++MO4n z&-P+j`GUX$jsOg*kqE?6qM$g-yE-VI8E;CZf`X&NmDk=dWvRUr^|_=PB#$X3?JZRa z!*g3|@5H+~ps?rN91{+*M?_Uy;UJZTh=^!tZ|YW?L3>ANZ)Sbb-iZ%y!MQ%X1@`T0 zmehFdoz$G2*n{UC9U@veWMR-P$Y@s}GD2Gx7Ds|Kl#ePMO0^fnKT zSL61Dib7%3c#87qPe#n1ywhe+UNaTFR#1Z*}Jh6H|tSa|w!4N(-v6>P!+sA3op z(}sFOVBYw&2P>5bC>LY~?*jN1;=jz)oNuOyEbf16`y7*e^(xoJ-_mxK7*RZaXE;>eH|=nw&9bD^3S z{|a~Zf}?!nEkO?r$>VU&Hh+! z=Sl5P(a+-;utLgl40v2uplo1)g4V_%i?BLyFalb$I%uthtqu$(9m5ZK6<*`;Id^O( zd`d>gK3++RijA=AMu#DT|0ibirbjTM5;UmsN?*K=`&O79hQ1MFS=$0bTzZ_fu|Y#S zMbVkT){p^Pqr5w84I0kd8az(3HAa!P6rMOwoyB(g7KnpN0W+p1-!wQu2#3mOEG4|a zQtUxS@p46yvISFnBw|XL=0vjHRzFH!HCKGuHh!5Z1{76qSxXFaaaIEb5j~?=;${<>L-0%} zS3jFY+6NSg0`bq83UtHV5>dxd1QH4J(2WB!>DuL*>M&e&U0lQ%8Q}z=#wrIlRz36#I@^PSWhj(2JY!AO8Zgysdb05M zeFJpf2wC+8UARzhe62!+XDcDWhPh^6#RSr6%uu!~L{K)CAq(AyTkth(HIhU$SOhU= z7=@tol~1hzibP7GH?=q6k=XKD2p2 zUyJO4ugc4gFHSi%D2mA<=u{|ZS4;!)EYmajgw-@4?i?(lGTz-maA69)_qN>xnKK>X z8LAg<2=q{~Z$=G;ND^2s2ORCf;{a(?H$LDXpfomgw{ViiVFZ!*CWMo?!sAdMnAV>-mT@oDJxsLne+yMv6_6)@aLD7e{h2uHg+_sp)5(dYshaY@3&Hh9%L z)3ZD5?DVLoN4pCqAcBK~gO3oBIF2!gfjEGTA=qFOvOcgeq2eUvBF<5XD^5~Hjv^G5 z6d?|k-}hd>*|#lm7>i1!;jeyQzx#XN``-7y*FC$;DQfIyVj_ElzmpZ&%{+_jX8C^0 zwwtxh5*ab7M-nfzdY{6!n}z8^xacCgS!^3)HzNajdaC7>)*;2lZpMWuE3SGIWR0dMp)@q0F$^%2~{b0}N&P%+l!I9%EQ{$Qfzf*E5t=4Jw^6 zl&K?9i&BKqBSrQjHYD_;5khmqLESI>Yv0Lg2p{2c&8tcf?(VK8_wwZUgP%`}+MLd*3 zL0(a`%21ZgzcN5&da6h#slU}>%Jb6{qU#YbIced>DX8>pm9)Va%A&vU;ziz>KD;x$ zSB-Zjsl78vu-G6u668MV%e1ghGX z)hB%yPRJCJITEo)wNqZQAx>JR>|@QP!NCvn9(1HGSj|O1B8ygWy0Vt)`Nl}>N^zB{ zTR*QWvMgM86=Xjaa0MpM$iY=wbyV!u)CH$d&RyzMNjY0_N2+L5EiKVskv*8~TcK*F zaq?;;*<{63Mbi-Fi>o-NC1FsK^+_><=W_fh<|cMCp63HiXQHIMW#;n`Kl{!{r|R|H zA@$ZRjg>#wWtK`Rb+jF=U4ZIOKgtNP7R%8G`&yDP+KHLH;<8FgP&5f%?h8&4A3ip8 zaz;6fWc7xV%mJ~IiJ=(W`Pwv+$7*Hn8QGEqAE){&AI4TdEh+ivC889HSe}R{+Mp0_ zFoX1(htVZH+pDbh6z_FNOGKuOQ7`qOg~c}R6T4r#?5QATuQ2pc@$eHDUVGU$zRAt-rp ze?&D;ed@bx|3yFgkYV+iKR-X9US_C#n<=E5Ee0SQOFb;vwfS{XLi&DW;#`C;G=2dZ zE&Q)sw?lcWh}yT2cNf&BaPXtIpSoT0 z%NNIVSbd{BSouyOb-!+#V$e0C5#fcD=_JT5_OsmZ)`?zJrV8pw zTzjIt#GqFt$;rLebpP6~K1LQ^Q<(aR3^gU_uQ&@3NL?_RDcW18G*Z5nw1k$#t0K9Q z>8q7YGE!HnSuz5&4OjPGmJunm!lZtaDdEiPW^(UjWXbQexI7nK)OH)7HLD54D>%j`dUSnvGeU1hgg`~)9Qu?fPA<#}5{d7qFHabrkIL&VvOl>_-k2v6ofYVNYWPmt) znu1c3zo?F6^wTi?<07;}wCJ13kTf;G_cxV6_)huU*k3D#;ftUVGwQcbmNOp~(K6YN z=<$lu3xi)HQ^a8Px8a2o!q12sdO|o%gdXR0|M`%YT7*|{AmaVeRLSiBvP`)(PGT^v zh5~JDH0e-Z^!1xoFQRkEZML-igWvCCs!&;7wSbW|@32rGIehK*1H3spLulh(us5MeJZAiz=b9 zgE#eQ79d>W*wft-@ga?97uj4`+Cy}m;xhxF)eHx$OlzfGE792<(?A-mDi+32aa4p< zETzg)N(m~Uikab0C-1;sm9L8iAK3BSJ5q^iB9YkJmq=X5-O&FCeUU|93cb~$E6`&W zeGBw9i+(CbUkiu@lfcPfHCO{q0c*j}fm6XdfdfthL*VDZ>EH}-CO8YM z180MGfnNaY0RxyA0iz%dG9Vk#x#*tfUI0ZOLZ|Iky(u^JREbvg16J6n z2mY+nOKP{oPGvRFq2_vHs!Ju(MRomu8%LQF=Y#Fwy+G!G$hpX>%+HI#CE!x96YK)J z!DZlm;BxSb;QinO;Dg`_a3%NZ4wm0%DY4^{zltso%J1+?Kw@a8v&6B1 z+nVxp!O`=@Atx$D$JPD7(T$*B3Y!Y2$(YiSG~lPp=5{(>5H|dlv#06R=}5{68dKBZ z9;fa$nqF<(Sv+FEp`LoEgDRF_;<#?hn{Rv75Dfq$!Y=(=Oq^eJTP;6y+-lY9zzb*h zG-{&e?C`5@)7cgJflL4Pxy`QU)Ve{VHRaGBw;pZ9$zpI^{AEr=7-&O8>;38bB&hpUk#buh5M4b zHz;-kbd?i~lG}WvCLFS%sx??FLRPKd?H#VBKQ$HA@fOFb>Ia@;AmTgP>dqdty>G+N z=wTUE!}505xim+afv5si-&^FD5v{h{YE)-TVTWSQPrIGz$-tj$4W~yk#gU8_Yjjw*ts?Sxc25ZpbdCuel3Nug=WL zrr^GT#A=g;^~#eDsFHeBlj0WkOJa}RjFEB__o|ES+6WgMf1elBo4(jM3xg?TldUBz z_DbDtc05))FP%3k-i&FAv@d?8=Vm-6NOSRq}=6taa} zAzvsIiiJ|4To@~+imP(~^X{?+sXUf@f zuADCy%EfZ2TrQ7|(Zn&TXLY2YF%pf*KI6h8Qpy&+96m@N)0u28UnrK!WA0?N=GD{l zOHV4BHgDOwZM`$Feqi0Y0onB|*&5v%YJ*=TqvG`-5b5{D}WAP_L%RbrUUlWU83%$vTKNVW`#wNWJiywlPJ++BH zJ&N~R6H`IEI#F%<9d8_)TeB?mXS@~+TaV@2@>|2Ouw_hSMr_t$F>zELi>LfnRN@%w zP{!qiY*xp|mkgDqx{r(T`lJ}IpN6)1{S36t>t~^DUjGTS-M2rD#s4|9&Fde;;(rKj z^ZKu1@qYuo$>QDLLfgFlQ7r!Nq3!nkLlkd#%}U`l_IVSHRzp^fiCHgnYi{Ts$o?gs zJx>|JTQhzo==B!;ZfILxWv?qbyw+{A(PJNg#@9o-REF$*?L6OrmUhoJ^is1~oE!7u zZhRXK8>_(g!-;4v_1f`0(ysw(!N!(pOssMqq-Bk%`Yj+wr6Q+DB;OGtjnv$U)ol zy%3A%qh-|}lYcB0e+Bgwt@`)G>TeNew_p50cKc);?Dh-4Em>j?pg~_lnoOHy+-zBr zak1;X6WXrxGqh3Sja-}AZp!;?6mRIyLEC-26WZqC-Ox4RnAy>|IOhPKQ5CulqWKSSFx@n^C8KZxc3i&*}@j^!Vqy|xW^6trFb z3#4~|;m^Mlw&&Q3(4u2a{7Zx-zqu$baUb|SAok|-*qgG>N}eImJ6|Gvfkpot;fpPL zHQ}=?TGr4rE&4RVvXe07-AY*aW@ynRcK&Zci~VBa|A=tLqGe13CjQHW?e@sJwZn@4 zL@d4zz0->S5aIV)^!o^lJ!Z-meQ5XpD(G{qcv&;fv*@o9KHs7z314E-A0RC2h$(*` z;Y%%A^zl}UmND98(O)OL8;l>GQa*p5u&k|yz9~k_+_vMzp0w#3V)P%z^8YvV8ms*N zSo*(=rQc4x$dGA|=suf%DXG_c6aNad-5#08HZ6Mzn-<>K^pmmt7@djHPsG}nh{Zo2iG#FxN{qfeMi&oT z-o7(q@hfBT55@W`z7)HEn`7x;hOSuh@juWqrbZrKg_bf5-M2!|M??2Ri@j**70{w5 z4gC)2cUg1_+OF>e=&e@#iO_a^tD$B7oATB`+x4x59JKYduA?1djZn^>COz?p)dIiWmSfOYncp{1)(jB6{b~F-9qxa}c#F>c z0$2rLSK{}9wBrF_E|Mi41g^6M_j=P^a2m6m2+ZQ((@w&k;qSH_4@X7dES()S7ZOJf zq|{kqeNfwCQm)h`_#%*V4?4GJr`PE=!}0NMYc6ox>(1UIyDN&^vo#3z=rbi!>r2}7 zCGZfC^Fk?0^fyWUV3!x}P!4)UT|9Gm0Mt<+nxx0PpY2a#BLjh*1ZJE zb*iFdSG3EY_0GrN?5GnK=Ri;5Gw`ZLKh6Lmi*oT6v$`JQJk!3`u^8U=>@zPNKJd=k zz{B_d=gr=aubjQ|kDptA$-h+REB9W$_mq!4?mC<&(gW?Cm4mD|9=y3}jxgom5|(*E zb3uej8|O}qtt-cWth@L-x6_!9GN+H>{xC2y^a$ZU0FTC=e+Bxh;4z?2A4Q<&f(E&-54_2Y;(}YAk%KYH?)kmP2jB(XAsJOpH$%jTBhdh_z-)s#i~IqfdhKd`*9 zOJg9-eO55- z`!o7g6*oF!knFl>>NEZMBw-mVxuji!?*LVZNGqoj6?LwzR0}an{rfKA>w)3J(}ZPS mNdHl*hxDBZDNppPl%J?P@GLKl9WV3`k4~xQC#^~;o%Fw0j-YA) literal 0 HcmV?d00001 From 520f216ca11d34f2eb114a2d891c7ec1ab0ad68b Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Fri, 15 May 2026 15:20:31 +0800 Subject: [PATCH 04/19] Add Higress WasmPlugin OCI support --- crates/config/Cargo.toml | 1 + crates/config/src/service/k8s/convert.rs | 1 + .../k8s/convert/higress_wasm_plugin_conv.rs | 407 ++++++++++++++++++ crates/config/src/service/k8s/listen.rs | 101 ++++- crates/config/src/service/k8s/retrieve.rs | 154 ++++++- crates/model/src/ext/k8s/crd.rs | 1 + crates/model/src/ext/k8s/crd/wasm_plugin.rs | 71 +++ crates/plugin-wasm/Cargo.toml | 1 + crates/plugin-wasm/src/abi.rs | 15 +- crates/plugin-wasm/src/config.rs | 52 ++- crates/plugin-wasm/src/fetch.rs | 356 ++++++++++++++- crates/plugin-wasm/src/host_fn.rs | 362 ++++++---------- crates/plugin-wasm/src/host_state.rs | 5 +- crates/plugin-wasm/src/runtime.rs | 46 +- crates/plugin-wasm/src/shared.rs | 9 +- crates/plugin-wasm/src/shell.rs | 2 +- crates/plugin-wasm/src/vm.rs | 122 +++--- crates/plugin-wasm/tests/http_call.rs | 9 +- crates/plugin-wasm/tests/on_tick.rs | 2 +- crates/plugin-wasm/tests/runtime_fetch.rs | 192 +++++++++ crates/plugin-wasm/tests/sdk_examples.rs | 31 +- crates/plugin-wasm/tests/spec_compliance.rs | 12 +- docs/k8s/gateway-api-compatibility.md | 27 ++ examples/wasm-hello/README.md | 18 + .../higress-wasmplugin-crd.yaml | 81 ++++ .../spacegate-admin-server.yaml | 22 +- .../kube-manifests/spacegate-gateway.yaml | 15 + .../wasmplugin-hello-example.yaml | 19 + .../plugin/wasm.hello-world.json | 3 + 29 files changed, 1747 insertions(+), 390 deletions(-) create mode 100644 crates/config/src/service/k8s/convert/higress_wasm_plugin_conv.rs create mode 100644 crates/model/src/ext/k8s/crd/wasm_plugin.rs create mode 100644 crates/plugin-wasm/tests/runtime_fetch.rs create mode 100644 resource/kube-manifests/higress-wasmplugin-crd.yaml create mode 100644 resource/kube-manifests/wasmplugin-hello-example.yaml diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index 2bb92beb..11e08e90 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -47,6 +47,7 @@ tokio-rustls.workspace = true ipnet = { workspace = true, features = ["serde"] } bytes = { workspace = true } +base64 = { workspace = true } kube = { workspace = true, optional = true } k8s-openapi = { workspace = true, optional = true } diff --git a/crates/config/src/service/k8s/convert.rs b/crates/config/src/service/k8s/convert.rs index 875d6b79..a88ddda6 100644 --- a/crates/config/src/service/k8s/convert.rs +++ b/crates/config/src/service/k8s/convert.rs @@ -2,6 +2,7 @@ use spacegate_model::ext::k8s::crd::sg_filter::K8sSgFilterSpecTargetRef; pub mod filter_k8s_conv; pub mod gateway_k8s_conv; +pub mod higress_wasm_plugin_conv; pub mod route_k8s_conv; pub(crate) trait ToTarget { diff --git a/crates/config/src/service/k8s/convert/higress_wasm_plugin_conv.rs b/crates/config/src/service/k8s/convert/higress_wasm_plugin_conv.rs new file mode 100644 index 00000000..06f32552 --- /dev/null +++ b/crates/config/src/service/k8s/convert/higress_wasm_plugin_conv.rs @@ -0,0 +1,407 @@ +use kube::ResourceExt; +use serde_json::{json, Map, Value}; +use spacegate_model::{ + ext::k8s::crd::wasm_plugin::{HigressWasmPluginMatchRule, WasmPlugin}, + BackendHost, PluginConfig, PluginInstanceId, PluginInstanceName, SgBackendRef, +}; + +const WASM_CODE: &str = "wasm"; + +pub(crate) trait HigressWasmPluginConv { + fn to_spacegate_plugin_id(&self) -> PluginInstanceId; + fn to_spacegate_rule_plugin_id(&self, rule_index: usize) -> PluginInstanceId; + fn to_spacegate_plugin_configs(&self) -> Vec; + fn to_spacegate_plugin_configs_with_oci_auth(&self, oci_auth: Option) -> Vec; + fn to_spacegate_plugin_config_by_id(&self, id: &PluginInstanceId) -> Option; + fn to_spacegate_plugin_config_by_id_with_oci_auth(&self, id: &PluginInstanceId, oci_auth: Option) -> Option; + fn gateway_plugin_id(&self) -> Option; + fn route_plugin_ids(&self, route_name: &str, hostnames: Option<&[String]>) -> Vec; + fn backend_plugin_ids

(&self, backend: &SgBackendRef

) -> Vec; + fn priority(&self) -> i32; + fn phase_rank(&self) -> i32; + fn validate_for_spacegate(&self) -> Result<(), String>; + fn digest(&self) -> Option; + fn oci_registry(&self) -> Option; +} + +impl HigressWasmPluginConv for WasmPlugin { + fn to_spacegate_plugin_id(&self) -> PluginInstanceId { + PluginInstanceId { + code: WASM_CODE.into(), + name: PluginInstanceName::named(format!("higress-{}", self.name_any())), + } + } + + fn to_spacegate_rule_plugin_id(&self, rule_index: usize) -> PluginInstanceId { + PluginInstanceId { + code: WASM_CODE.into(), + name: PluginInstanceName::named(format!("higress-{}-rule-{rule_index}", self.name_any())), + } + } + + fn to_spacegate_plugin_configs(&self) -> Vec { + self.to_spacegate_plugin_configs_with_oci_auth(None) + } + + fn to_spacegate_plugin_configs_with_oci_auth(&self, oci_auth: Option) -> Vec { + let mut configs = Vec::new(); + if !self.spec.default_config_disable { + configs.push(build_plugin_config( + self, + self.to_spacegate_plugin_id(), + self.spec.default_config.clone(), + "default", + oci_auth.clone(), + )); + } + configs.extend(self.spec.match_rules.iter().enumerate().filter(|(_, rule)| !rule.config_disable).map(|(idx, rule)| { + build_plugin_config( + self, + self.to_spacegate_rule_plugin_id(idx), + build_higress_rule_config(rule), + &format!("rule-{idx}"), + oci_auth.clone(), + ) + })); + configs + } + + fn to_spacegate_plugin_config_by_id(&self, id: &PluginInstanceId) -> Option { + self.to_spacegate_plugin_config_by_id_with_oci_auth(id, None) + } + + fn to_spacegate_plugin_config_by_id_with_oci_auth(&self, id: &PluginInstanceId, oci_auth: Option) -> Option { + self.to_spacegate_plugin_configs_with_oci_auth(oci_auth).into_iter().find(|cfg| &cfg.id == id) + } + + fn gateway_plugin_id(&self) -> Option { + (!self.spec.default_config_disable).then(|| self.to_spacegate_plugin_id()) + } + + fn route_plugin_ids(&self, route_name: &str, hostnames: Option<&[String]>) -> Vec { + self.spec + .match_rules + .iter() + .enumerate() + .filter(|(_, rule)| !rule.config_disable && rule_matches_route(rule, route_name, hostnames)) + .map(|(idx, _)| self.to_spacegate_rule_plugin_id(idx)) + .collect() + } + + fn backend_plugin_ids

(&self, backend: &SgBackendRef

) -> Vec { + self.spec + .match_rules + .iter() + .enumerate() + .filter(|(_, rule)| !rule.config_disable && rule_matches_backend(rule, backend)) + .map(|(idx, _)| self.to_spacegate_rule_plugin_id(idx)) + .collect() + } + + fn priority(&self) -> i32 { + self.spec.priority.unwrap_or(0) + } + + fn phase_rank(&self) -> i32 { + phase_rank(self.spec.phase.as_deref()) + } + + fn validate_for_spacegate(&self) -> Result<(), String> { + let url = self.spec.url.trim(); + if url.is_empty() { + return Err("spec.url is empty".to_string()); + } + if is_oci_url(url) && parse_oci_registry(url).is_none() { + return Err("spec.url must include OCI registry and repository".to_string()); + } + Ok(()) + } + + fn digest(&self) -> Option { + self.spec.sha256.clone() + } + + fn oci_registry(&self) -> Option { + parse_oci_registry(&self.spec.url) + } +} + +fn build_plugin_config(plugin: &WasmPlugin, id: PluginInstanceId, plugin_config: Value, instance_suffix: &str, oci_auth: Option) -> PluginConfig { + let namespace = plugin.namespace().unwrap_or_else(|| "default".to_string()); + let resource_version = plugin.resource_version().unwrap_or_else(|| "unknown".to_string()); + let plugin_name = plugin.spec.plugin_name.clone().unwrap_or_else(|| plugin.name_any()); + let image_pull_always = plugin.spec.image_pull_policy.as_deref().map(|v| v.eq_ignore_ascii_case("always")).unwrap_or(false); + + let mut spec = json!({ + "url": plugin.spec.url, + "plugin_config": plugin_config, + "plugin_name": plugin_name, + "plugin_root_id": format!("higress-{}-root-{instance_suffix}", plugin.name_any()), + "plugin_vm_id": format!("higress-{}-{}-{instance_suffix}", namespace, plugin.name_any()), + "module_cache_key": format!("higress-wasmplugin:{namespace}:{}:{resource_version}:{instance_suffix}", plugin.name_any()), + "use_cache": !image_pull_always, + }); + + if let Some(sha256) = plugin.spec.sha256.as_deref().filter(|v| !v.trim().is_empty()) { + spec["sha256"] = Value::String(sha256.to_string()); + } + if let Some(fail_strategy) = plugin.spec.fail_strategy.as_deref().and_then(normalize_fail_strategy) { + spec["fail_strategy"] = Value::String(fail_strategy.to_string()); + } + if let Some(oci_auth) = oci_auth { + spec["oci_auth"] = oci_auth; + } + + PluginConfig { id, spec } +} + +pub(crate) fn sort_higress_wasm_plugins(plugins: &mut [WasmPlugin]) { + plugins.sort_by(|a, b| a.phase_rank().cmp(&b.phase_rank()).then_with(|| b.priority().cmp(&a.priority())).then_with(|| a.name_any().cmp(&b.name_any()))); +} + +fn normalize_fail_strategy(value: &str) -> Option<&'static str> { + match value.trim().to_ascii_lowercase().replace('-', "_").as_str() { + "fail_open" | "failopen" => Some("fail_open"), + "fail_close" | "failclose" => Some("fail_close"), + _ => None, + } +} + +fn build_higress_rule_config(rule: &HigressWasmPluginMatchRule) -> Value { + let mut config = value_to_object(rule.config.clone()); + if !rule.ingress.is_empty() { + config.insert("_match_route_".to_string(), strings_value(rule.ingress.clone())); + } + if !rule.domain.is_empty() { + config.insert("_match_domain_".to_string(), strings_value(rule.domain.clone())); + } + if !rule.service.is_empty() { + config.insert("_match_service_".to_string(), strings_value(rule.service.clone())); + } + if rule.config_disable { + config.insert("_config_disable_".to_string(), Value::Bool(true)); + } + Value::Object(config) +} + +fn value_to_object(value: Value) -> Map { + match value { + Value::Object(map) => map, + Value::Null => Map::new(), + other => { + let mut map = Map::new(); + map.insert("_config_".to_string(), other); + map + } + } +} + +fn strings_value(values: Vec) -> Value { + Value::Array(values.into_iter().map(Value::String).collect()) +} + +fn phase_rank(phase: Option<&str>) -> i32 { + match phase.unwrap_or_default().trim().to_ascii_uppercase().as_str() { + "AUTHN" => 10, + "AUTHZ" => 20, + "STATS" => 90, + _ => 50, + } +} + +fn rule_matches_route(rule: &HigressWasmPluginMatchRule, route_name: &str, hostnames: Option<&[String]>) -> bool { + let route_match = !rule.ingress.is_empty() && rule.ingress.iter().any(|name| name.eq_ignore_ascii_case(route_name)); + let domain_match = + !rule.domain.is_empty() && hostnames.map(|hostnames| hostnames.iter().any(|hostname| rule.domain.iter().any(|domain| domain_matches(domain, hostname)))).unwrap_or(false); + let rule_has_no_explicit_target = rule.ingress.is_empty() && rule.domain.is_empty() && rule.service.is_empty(); + route_match || domain_match || rule_has_no_explicit_target +} + +fn rule_matches_backend

(rule: &HigressWasmPluginMatchRule, backend: &SgBackendRef

) -> bool { + !rule.service.is_empty() && rule.service.iter().any(|service| backend_matches_service(backend, service)) +} + +fn domain_matches(pattern: &str, hostname: &str) -> bool { + let pattern = pattern.trim().trim_end_matches('.'); + let hostname = hostname.trim().trim_end_matches('.'); + if pattern == "*" { + return true; + } + if let Some(suffix) = pattern.strip_prefix("*.") { + return hostname.eq_ignore_ascii_case(suffix) || hostname.to_ascii_lowercase().ends_with(&format!(".{}", suffix.to_ascii_lowercase())); + } + pattern.eq_ignore_ascii_case(hostname) +} + +fn backend_matches_service

(backend: &SgBackendRef

, service: &str) -> bool { + let service = service.trim(); + match &backend.host { + BackendHost::K8sService(data) => { + data.name.eq_ignore_ascii_case(service) || data.namespace.as_ref().map(|ns| format!("{}.{}", data.name, ns).eq_ignore_ascii_case(service)).unwrap_or(false) + } + _ => backend.get_host().eq_ignore_ascii_case(service), + } +} + +fn is_oci_url(url: &str) -> bool { + let lower = url.to_ascii_lowercase(); + lower.starts_with("oci://") || lower.starts_with("docker://") || lower.starts_with("image://") || lower.starts_with("oci+http://") +} + +fn parse_oci_registry(url: &str) -> Option { + let trim = url.trim(); + let rest = trim.strip_prefix("oci://").or_else(|| trim.strip_prefix("docker://")).or_else(|| trim.strip_prefix("image://")).or_else(|| trim.strip_prefix("oci+http://"))?; + let (registry, repository) = rest.split_once('/')?; + (!registry.trim().is_empty() && !repository.trim().is_empty()).then(|| registry.to_string()) +} + +#[cfg(test)] +mod tests { + use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; + use serde_json::json; + use spacegate_model::{ + ext::k8s::crd::wasm_plugin::{HigressWasmPluginSpec, WasmPlugin}, + BackendHost, K8sServiceData, SgBackendRef, + }; + + use super::*; + + #[test] + fn converts_higress_wasmplugin_to_spacegate_wasm_plugin_config() { + let plugin = WasmPlugin { + metadata: ObjectMeta { + name: Some("authn".to_string()), + namespace: Some("gw".to_string()), + resource_version: Some("42".to_string()), + ..Default::default() + }, + spec: HigressWasmPluginSpec { + url: "https://example.com/authn.wasm".to_string(), + plugin_name: Some("authn-plugin".to_string()), + sha256: Some("sha256:abc".to_string()), + phase: Some("AUTHN".to_string()), + priority: Some(100), + image_pull_policy: Some("IfNotPresent".to_string()), + image_pull_secret: None, + default_config_disable: false, + default_config: json!({"issuer": "spacegate"}), + match_rules: vec![HigressWasmPluginMatchRule { + ingress: vec!["api-route".to_string()], + domain: vec!["api.example.com".to_string()], + service: vec![], + config_disable: false, + config: json!({"issuer": "route"}), + }], + fail_strategy: Some("FAIL_CLOSE".to_string()), + }, + status: None, + }; + + let cfg = plugin.to_spacegate_plugin_configs().into_iter().next().expect("default config"); + assert_eq!(cfg.id.to_string(), "wasm-n-higress-authn"); + assert_eq!(cfg.spec["url"], "https://example.com/authn.wasm"); + assert_eq!(cfg.spec["sha256"], "sha256:abc"); + assert_eq!(cfg.spec["fail_strategy"], "fail_close"); + assert_eq!(cfg.spec["module_cache_key"], "higress-wasmplugin:gw:authn:42:default"); + assert_eq!(cfg.spec["plugin_config"]["issuer"], "spacegate"); + + let rule_cfg = plugin.to_spacegate_plugin_config_by_id(&plugin.to_spacegate_rule_plugin_id(0)).expect("rule config"); + assert_eq!(rule_cfg.id.to_string(), "wasm-n-higress-authn-rule-0"); + assert_eq!(rule_cfg.spec["plugin_config"]["issuer"], "route"); + assert_eq!(rule_cfg.spec["plugin_config"]["_match_route_"][0], "api-route"); + assert_eq!(rule_cfg.spec["plugin_config"]["_match_domain_"][0], "api.example.com"); + } + + #[test] + fn rule_ids_match_routes_domains_and_services() { + let plugin = WasmPlugin { + metadata: ObjectMeta { + name: Some("ratelimit".to_string()), + namespace: Some("gw".to_string()), + resource_version: Some("7".to_string()), + ..Default::default() + }, + spec: HigressWasmPluginSpec { + url: "file:///tmp/ratelimit.wasm".to_string(), + plugin_name: None, + sha256: None, + phase: Some("AUTHZ".to_string()), + priority: Some(10), + image_pull_policy: None, + image_pull_secret: None, + default_config_disable: true, + default_config: serde_json::Value::Null, + match_rules: vec![ + HigressWasmPluginMatchRule { + ingress: vec!["api-route".to_string()], + domain: vec![], + service: vec![], + config_disable: false, + config: json!({"limit": 10}), + }, + HigressWasmPluginMatchRule { + ingress: vec![], + domain: vec!["*.example.com".to_string()], + service: vec![], + config_disable: false, + config: json!({"limit": 20}), + }, + HigressWasmPluginMatchRule { + ingress: vec![], + domain: vec![], + service: vec!["backend.default".to_string()], + config_disable: false, + config: json!({"limit": 30}), + }, + ], + fail_strategy: None, + }, + status: None, + }; + + let route_ids = plugin.route_plugin_ids("api-route", Some(&["shop.example.com".to_string()])); + assert_eq!( + route_ids.iter().map(ToString::to_string).collect::>(), + vec!["wasm-n-higress-ratelimit-rule-0", "wasm-n-higress-ratelimit-rule-1"] + ); + + let backend = SgBackendRef:: { + host: BackendHost::K8sService(K8sServiceData { + name: "backend".to_string(), + namespace: Some("default".to_string()), + }), + ..Default::default() + }; + let backend_ids = plugin.backend_plugin_ids(&backend); + assert_eq!(backend_ids.iter().map(ToString::to_string).collect::>(), vec!["wasm-n-higress-ratelimit-rule-2"]); + } + + #[test] + fn validates_oci_urls_for_status() { + let plugin = WasmPlugin { + metadata: ObjectMeta { + name: Some("oci-plugin".to_string()), + namespace: Some("gw".to_string()), + resource_version: Some("1".to_string()), + ..Default::default() + }, + spec: HigressWasmPluginSpec { + url: "oci://registry.example.com/plugin:v1".to_string(), + plugin_name: None, + sha256: None, + phase: None, + priority: None, + image_pull_policy: Some("Always".to_string()), + image_pull_secret: Some("pull-secret".to_string()), + default_config_disable: false, + default_config: serde_json::Value::Null, + match_rules: vec![], + fail_strategy: None, + }, + status: None, + }; + + plugin.validate_for_spacegate().expect("OCI should be accepted"); + assert_eq!(plugin.oci_registry().as_deref(), Some("registry.example.com")); + } +} diff --git a/crates/config/src/service/k8s/listen.rs b/crates/config/src/service/k8s/listen.rs index 9cb21d72..569e9ba5 100644 --- a/crates/config/src/service/k8s/listen.rs +++ b/crates/config/src/service/k8s/listen.rs @@ -6,8 +6,9 @@ use std::{ use futures_util::{pin_mut, TryStreamExt}; use k8s_gateway_api::{Gateway, HttpRoute}; +use k8s_openapi::api::core::v1::Secret; use kube::{ - api::ObjectMeta, + api::{ObjectMeta, PostParams}, runtime::{watcher, WatchStreamExt}, Api, Resource, ResourceExt, }; @@ -16,12 +17,19 @@ use spacegate_model::{ ext::k8s::crd::{ http_spaceroute::HttpSpaceroute, sg_filter::{K8sSgFilterSpecTargetRef, SgFilter}, + wasm_plugin::{HigressWasmPluginStatus, WasmPlugin}, }, BoxResult, Config, PluginInstanceId, }; use tracing::debug; -use crate::service::{k8s::convert::filter_k8s_conv::PluginIdConv, ConfigEventType, ConfigType, CreateListener, Listen, ListenEvent, Retrieve as _}; +use crate::service::{ + k8s::{ + convert::{filter_k8s_conv::PluginIdConv, higress_wasm_plugin_conv::HigressWasmPluginConv as _}, + retrieve::oci_auth_from_secret, + }, + ConfigEventType, ConfigType, CreateListener, Listen, ListenEvent, Retrieve as _, +}; use super::K8s; @@ -31,6 +39,43 @@ pub struct K8sListener { impl K8sListener {} impl K8s { + async fn reconcile_wasm_plugin_status(api: &Api, secret_api: &Api, plugin: &WasmPlugin) { + let (phase, message) = match Self::validate_wasm_plugin(api, secret_api, plugin).await { + Ok(()) => ("Accepted".to_string(), "WasmPlugin accepted by Spacegate".to_string()), + Err(e) => ("Unsupported".to_string(), e), + }; + let status = HigressWasmPluginStatus { + observed_generation: plugin.meta().generation, + phase: Some(phase), + digest: plugin.digest(), + message: Some(message), + }; + let mut update = plugin.clone(); + update.status = Some(status); + if let Err(e) = api.replace_status(&plugin.name_any(), &PostParams::default(), serde_json::to_vec(&update).unwrap_or_default()).await { + tracing::warn!(name = %plugin.name_any(), error = %e, "failed to update WasmPlugin status"); + } + } + + async fn validate_wasm_plugin(_api: &Api, secret_api: &Api, plugin: &WasmPlugin) -> Result<(), String> { + plugin.validate_for_spacegate()?; + let Some(registry) = plugin.oci_registry() else { + return Ok(()); + }; + let Some(secret_name) = plugin.spec.image_pull_secret.as_deref().map(str::trim).filter(|v| !v.is_empty()) else { + return Ok(()); + }; + let secret = secret_api + .get_opt(secret_name) + .await + .map_err(|e| format!("read imagePullSecret {secret_name}: {e}"))? + .ok_or_else(|| format!("imagePullSecret {secret_name} not found"))?; + if oci_auth_from_secret(&secret, ®istry).is_none() { + return Err(format!("imagePullSecret {secret_name} does not contain credentials for {registry}")); + } + Ok(()) + } + async fn process_http_spaceroute_event( move_evt_tx: &tokio::sync::mpsc::UnboundedSender<(ConfigType, ConfigEventType)>, move_http_route_names: &[String], @@ -111,6 +156,8 @@ impl CreateListener for K8s { let http_route_api: Api = self.get_namespace_api(); let http_spaceroute_api: Api = self.get_namespace_api(); let sg_filter_api: Api = self.get_namespace_api(); + let wasm_plugin_api: Api = self.get_namespace_api(); + let secret_api: Api = self.get_namespace_api(); let move_gateway_names = config.gateways.clone().into_values().map(|item| item.gateway.name).collect::>(); let move_evt_tx = evt_tx.clone(); @@ -326,6 +373,56 @@ impl CreateListener for K8s { } }); + let move_evt_tx = evt_tx.clone(); + let wasm_plugin_status_api = wasm_plugin_api.clone(); + let wasm_plugin_secret_api = secret_api.clone(); + // watch Higress-compatible WasmPlugin. A WasmPlugin can add/remove gateway-level + // plugins, so the simplest correct reconciliation is a global reload. + tokio::task::spawn(async move { + let mut uid_version_map = HashMap::new(); + let ew = watcher::watcher(wasm_plugin_api, watcher::Config::default()); + pin_mut!(ew); + while let Some(event) = ew.try_next().await.unwrap_or_default() { + match event { + watcher::Event::Applied(plugin) => { + Self::reconcile_wasm_plugin_status(&wasm_plugin_status_api, &wasm_plugin_secret_api, &plugin).await; + if uid_version_map.get(&plugin.uid()) == Some(plugin.meta()) { + continue; + } + uid_version_map.insert(plugin.uid(), plugin.meta().clone()); + move_evt_tx + .send(( + ConfigType::Plugin { + id: plugin.to_spacegate_plugin_id(), + }, + ConfigEventType::Update, + )) + .expect("send event error"); + move_evt_tx.send((ConfigType::Global, ConfigEventType::Update)).expect("send event error"); + } + watcher::Event::Deleted(plugin) => { + uid_version_map.remove(&plugin.uid()); + move_evt_tx + .send(( + ConfigType::Plugin { + id: plugin.to_spacegate_plugin_id(), + }, + ConfigEventType::Delete, + )) + .expect("send event error"); + move_evt_tx.send((ConfigType::Global, ConfigEventType::Update)).expect("send event error"); + } + watcher::Event::Restarted(plugins) => { + for plugin in &plugins { + Self::reconcile_wasm_plugin_status(&wasm_plugin_status_api, &wasm_plugin_secret_api, plugin).await; + } + uid_version_map = plugins.into_iter().map(|plugin| (plugin.uid(), plugin.meta().clone())).collect(); + move_evt_tx.send((ConfigType::Global, ConfigEventType::Update)).expect("send event error"); + } + } + } + }); + let listener = K8sListener { rx: evt_rx }; Ok((config, listener)) diff --git a/crates/config/src/service/k8s/retrieve.rs b/crates/config/src/service/k8s/retrieve.rs index e8e75848..9ce0b939 100644 --- a/crates/config/src/service/k8s/retrieve.rs +++ b/crates/config/src/service/k8s/retrieve.rs @@ -1,14 +1,17 @@ +use base64::{engine::general_purpose, Engine as _}; use futures_util::future::join_all; use gateway::{SgListener, SgParameters, SgProtocolConfig, SgTlsConfig}; use http_route::SgHttpRouteRule; use k8s_gateway_api::{Gateway, HttpRoute, Listener}; use k8s_openapi::api::core::v1::Secret; use kube::{api::ListParams, Api, ResourceExt}; +use serde_json::{json, Value}; use spacegate_model::{ ext::k8s::{ crd::{ http_spaceroute::HttpSpaceroute, sg_filter::{K8sSgFilterSpecTargetRef, SgFilter}, + wasm_plugin::WasmPlugin, }, helper_struct::SgTargetKind, }, @@ -23,7 +26,12 @@ use crate::{ }; use super::{ - convert::{filter_k8s_conv::PluginConfigConv, gateway_k8s_conv::SgParametersConv as _, route_k8s_conv::SgHttpRouteRuleConv as _}, + convert::{ + filter_k8s_conv::PluginConfigConv, + gateway_k8s_conv::SgParametersConv as _, + higress_wasm_plugin_conv::{sort_higress_wasm_plugins, HigressWasmPluginConv as _}, + route_k8s_conv::SgHttpRouteRuleConv as _, + }, K8s, }; @@ -137,14 +145,42 @@ impl Retrieve for K8s { async fn retrieve_all_plugins(&self) -> Result, BoxError> { let filter_api: Api = self.get_namespace_api(); + let wasm_plugin_api: Api = self.get_namespace_api(); - let result = filter_api.list(&ListParams::default()).await?.into_iter().filter_map(PluginConfig::from_first_filter_obj).collect(); + let mut result = filter_api.list(&ListParams::default()).await?.into_iter().filter_map(PluginConfig::from_first_filter_obj).collect::>(); + let mut wasm_plugins = wasm_plugin_api.list(&ListParams::default()).await?.items; + sort_higress_wasm_plugins(&mut wasm_plugins); + for plugin in wasm_plugins { + let oci_auth = self.resolve_higress_wasm_oci_auth(&plugin).await?; + if oci_auth.is_some() { + result.extend(plugin.to_spacegate_plugin_configs_with_oci_auth(oci_auth)); + } else { + result.extend(plugin.to_spacegate_plugin_configs()); + } + } Ok(result) } async fn retrieve_plugin(&self, id: &spacegate_model::PluginInstanceId) -> Result, BoxError> { let filter_api: Api = self.get_namespace_api(); + let wasm_plugin_api: Api = self.get_namespace_api(); + if id.code == "wasm" { + if let spacegate_model::PluginInstanceName::Named { name } = &id.name { + if let Some(wasm_name) = name.strip_prefix("higress-") { + let wasm_name = wasm_name.rsplit_once("-rule-").map(|(base, _)| base).unwrap_or(wasm_name); + if let Some(plugin) = wasm_plugin_api.get_opt(wasm_name).await? { + let oci_auth = self.resolve_higress_wasm_oci_auth(&plugin).await?; + return Ok(if oci_auth.is_some() { + plugin.to_spacegate_plugin_config_by_id_with_oci_auth(id, oci_auth) + } else { + plugin.to_spacegate_plugin_config_by_id(id) + }); + } + return Ok(None); + } + } + } match &id.name { spacegate_model::PluginInstanceName::Anon { uid: _ } => Ok(None), spacegate_model::PluginInstanceName::Named { name } => { @@ -174,13 +210,14 @@ impl K8s { } async fn kube_gateway_2_sg_gateway(&self, gateway_obj: Gateway) -> BoxResult { let gateway_name = gateway_obj.name_any(); - let plugins = self + let mut plugins = self .retrieve_config_item_filters(K8sSgFilterSpecTargetRef { kind: SgTargetKind::Gateway.into(), name: gateway_name.clone(), namespace: gateway_obj.namespace(), }) .await?; + plugins.extend(self.retrieve_higress_gateway_plugins(gateway_obj.namespace()).await?); let result = SgGateway { name: gateway_name, parameters: SgParameters::from_kube_gateway(&gateway_obj), @@ -192,6 +229,7 @@ impl K8s { async fn kube_httpspaceroute_2_sg_route(&self, httpspace_route: HttpSpaceroute) -> BoxResult { let route_name = httpspace_route.name_any(); + let namespace = httpspace_route.namespace(); let kind = if let Some(kind) = httpspace_route.annotations().get(constants::RAW_HTTP_ROUTE_KIND) { kind.clone() } else { @@ -205,7 +243,7 @@ impl K8s { namespace: httpspace_route.namespace(), }) .await?; - Ok(SgHttpRoute { + let mut route = SgHttpRoute { hostnames: httpspace_route.spec.hostnames.clone(), plugins, rules: httpspace_route @@ -216,7 +254,9 @@ impl K8s { .unwrap_or_default(), priority, route_name, - }) + }; + self.apply_higress_wasm_route_plugins(&mut route, namespace).await?; + Ok(route) } async fn kube_httproute_2_sg_route(&self, http_route: HttpRoute) -> BoxResult { @@ -243,6 +283,7 @@ impl K8s { }) .flat_map(|filter_obj| PluginConfig::from_first_filter_obj(filter_obj).map(|f| f.into())) .collect(); + let plugin_ids = plugin_ids; if !plugin_ids.is_empty() { let mut filter_vec = String::new(); @@ -324,4 +365,107 @@ impl K8s { .into_iter() .collect() } + + async fn retrieve_higress_gateway_plugins(&self, namespace: Option) -> BoxResult> { + let namespace = namespace.unwrap_or_else(|| self.namespace.to_string()); + let wasm_plugin_api: Api = self.get_specify_namespace_api(&namespace); + let mut wasm_plugins = wasm_plugin_api.list(&ListParams::default()).await?.items; + sort_higress_wasm_plugins(&mut wasm_plugins); + Ok(wasm_plugins.into_iter().filter_map(|p| p.gateway_plugin_id()).collect()) + } + + async fn apply_higress_wasm_route_plugins(&self, route: &mut SgHttpRoute, namespace: Option) -> BoxResult<()> { + let namespace = namespace.unwrap_or_else(|| self.namespace.to_string()); + let wasm_plugin_api: Api = self.get_specify_namespace_api(&namespace); + let mut wasm_plugins = wasm_plugin_api.list(&ListParams::default()).await?.items; + sort_higress_wasm_plugins(&mut wasm_plugins); + let hostnames = route.hostnames.as_deref(); + + for plugin in wasm_plugins { + route.plugins.extend(plugin.route_plugin_ids(&route.route_name, hostnames)); + for rule in &mut route.rules { + for backend in &mut rule.backends { + backend.plugins.extend(plugin.backend_plugin_ids(backend)); + } + } + } + Ok(()) + } + + async fn resolve_higress_wasm_oci_auth(&self, plugin: &WasmPlugin) -> BoxResult> { + if plugin.oci_registry().is_none() { + return Ok(None); + } + let Some(secret_name) = plugin.spec.image_pull_secret.as_deref().map(str::trim).filter(|v| !v.is_empty()) else { + return Ok(None); + }; + let namespace = plugin.namespace().unwrap_or_else(|| self.namespace.to_string()); + let secret_api: Api = self.get_specify_namespace_api(&namespace); + let Some(secret) = secret_api.get_opt(secret_name).await? else { + tracing::warn!( + wasm_plugin = %plugin.name_any(), + namespace = %namespace, + secret = %secret_name, + "WasmPlugin imagePullSecret not found" + ); + return Ok(None); + }; + Ok(plugin.oci_registry().and_then(|registry| oci_auth_from_secret(&secret, ®istry))) + } +} + +pub(crate) fn oci_auth_from_secret(secret: &Secret, registry: &str) -> Option { + let data = secret.data.as_ref()?; + if let Some(bytes) = data.get(".dockerconfigjson").or_else(|| data.get(".dockercfg")) { + if let Some(auth) = oci_auth_from_docker_config(&bytes.0, registry) { + return Some(auth); + } + } + + let username = secret_data_string(secret, "username").or_else(|| secret_data_string(secret, "user"))?; + let password = secret_data_string(secret, "password").unwrap_or_default(); + Some(json!({ + "registry": registry, + "username": username, + "password": password, + })) +} + +fn oci_auth_from_docker_config(bytes: &[u8], registry: &str) -> Option { + let config: Value = serde_json::from_slice(bytes).ok()?; + let auths = config.get("auths").and_then(Value::as_object)?; + let entry = auths.get(registry).or_else(|| auths.get(&format!("https://{registry}"))).or_else(|| auths.get(&format!("http://{registry}"))).or_else(|| { + (registry == "docker.io").then(|| auths.get("https://index.docker.io/v1/").or_else(|| auths.get("index.docker.io")).or_else(|| auths.get("registry-1.docker.io")))? + })?; + + let identity_token = entry.get("identitytoken").or_else(|| entry.get("identity_token")).and_then(Value::as_str).map(str::to_string); + if let Some(identity_token) = identity_token.filter(|v| !v.trim().is_empty()) { + return Some(json!({ + "registry": registry, + "identity_token": identity_token, + })); + } + + let (username, password) = if let (Some(username), Some(password)) = ( + entry.get("username").and_then(Value::as_str).filter(|v| !v.trim().is_empty()), + entry.get("password").and_then(Value::as_str), + ) { + (username.to_string(), password.to_string()) + } else { + let auth = entry.get("auth").and_then(Value::as_str)?; + let decoded = general_purpose::STANDARD.decode(auth).ok()?; + let decoded = String::from_utf8(decoded).ok()?; + let (username, password) = decoded.split_once(':')?; + (username.to_string(), password.to_string()) + }; + + Some(json!({ + "registry": registry, + "username": username, + "password": password, + })) +} + +fn secret_data_string(secret: &Secret, key: &str) -> Option { + secret.data.as_ref().and_then(|data| data.get(key)).and_then(|bytes| String::from_utf8(bytes.0.clone()).ok()) } diff --git a/crates/model/src/ext/k8s/crd.rs b/crates/model/src/ext/k8s/crd.rs index 3cd76b8e..5364f7f2 100644 --- a/crates/model/src/ext/k8s/crd.rs +++ b/crates/model/src/ext/k8s/crd.rs @@ -1,2 +1,3 @@ pub mod http_spaceroute; pub mod sg_filter; +pub mod wasm_plugin; diff --git a/crates/model/src/ext/k8s/crd/wasm_plugin.rs b/crates/model/src/ext/k8s/crd/wasm_plugin.rs new file mode 100644 index 00000000..df7b072d --- /dev/null +++ b/crates/model/src/ext/k8s/crd/wasm_plugin.rs @@ -0,0 +1,71 @@ +use k8s_openapi::schemars::JsonSchema; +use kube::CustomResource; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[serde(rename_all = "camelCase")] +#[kube(kind = "WasmPlugin", group = "extensions.higress.io", version = "v1alpha1", namespaced, status = "HigressWasmPluginStatus")] +pub struct HigressWasmPluginSpec { + /// Higress-compatible wasm URL. Spacegate runtime supports local paths, + /// `file://`, `http(s)://`, and OCI image URLs such as `oci://registry/repo:tag`. + pub url: String, + /// Optional plugin name exposed to proxy-wasm guests. + #[serde(default)] + pub plugin_name: Option, + /// Optional SHA-256 digest for the wasm bytes. Accepts either plain hex or `sha256:`. + #[serde(default, alias = "sha256")] + pub sha256: Option, + /// Higress phase is kept for ordering/compatibility. Spacegate currently maps order by priority. + #[serde(default)] + pub phase: Option, + /// Higher priority plugins are placed earlier in the generated Spacegate plugin list. + #[serde(default)] + pub priority: Option, + /// `Always` disables Spacegate's in-process wasm module cache for this plugin. + #[serde(default)] + pub image_pull_policy: Option, + /// Optional Kubernetes Secret used for private OCI registries. + #[serde(default)] + pub image_pull_secret: Option, + /// Disable global/default config. Match rules can still enable per-rule configs. + #[serde(default)] + pub default_config_disable: bool, + /// Higress default plugin config. + #[serde(default)] + pub default_config: Value, + /// Optional match rules. These are passed through to Higress-style wasm plugins under `_rules_`. + #[serde(default)] + pub match_rules: Vec, + /// Optional fail strategy. `FAIL_OPEN`/`FAIL_CLOSE` and `fail_open`/`fail_close` are accepted. + #[serde(default)] + pub fail_strategy: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct HigressWasmPluginStatus { + #[serde(default)] + pub observed_generation: Option, + #[serde(default)] + pub phase: Option, + #[serde(default)] + pub digest: Option, + #[serde(default)] + pub message: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct HigressWasmPluginMatchRule { + #[serde(default)] + pub ingress: Vec, + #[serde(default)] + pub domain: Vec, + #[serde(default)] + pub service: Vec, + #[serde(default)] + pub config_disable: bool, + #[serde(default)] + pub config: Value, +} diff --git a/crates/plugin-wasm/Cargo.toml b/crates/plugin-wasm/Cargo.toml index a3a7d8c5..f5d92ac3 100644 --- a/crates/plugin-wasm/Cargo.toml +++ b/crates/plugin-wasm/Cargo.toml @@ -26,6 +26,7 @@ serde_yaml = "0.9" tracing = { workspace = true } thiserror = "1" once_cell = "1.19" +sha2 = "0.10" # host fn: dispatch_http_call 走 reqwest 异步客户端（用 0.12 与 spacegate 的 http=1 对齐） reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } diff --git a/crates/plugin-wasm/src/abi.rs b/crates/plugin-wasm/src/abi.rs index 94a186bc..5c94adf1 100644 --- a/crates/plugin-wasm/src/abi.rs +++ b/crates/plugin-wasm/src/abi.rs @@ -246,9 +246,7 @@ impl MemoryHelper { /// 从 caller 中拿到 `memory` export 的 helper（在每个 host fn 起始处调用）。 pub fn from_caller(caller: &mut Caller<'_, T>) -> Result { let Some(mem) = caller.get_export("memory").and_then(|e| e.into_memory()) else { - return Err(WasmHostError::AbiViolation( - "guest module has no `memory` export".to_string(), - )); + return Err(WasmHostError::AbiViolation("guest module has no `memory` export".to_string())); }; Ok(Self { memory: mem }) } @@ -276,10 +274,7 @@ impl MemoryHelper { let start = ptr as usize; let end = start.saturating_add(data.len()); if end > mem.len() { - return Err(WasmHostError::MemoryOob { - ptr, - len: data.len() as u32, - }); + return Err(WasmHostError::MemoryOob { ptr, len: data.len() as u32 }); } mem[start..end].copy_from_slice(data); Ok(()) @@ -390,11 +385,7 @@ fn u32_from_slice(bytes: &[u8], pos: usize) -> Option { /// /// spec §Serialization: "Host implementations should tolerate a NULL character at the end". pub fn decode_property_path(bytes: &[u8]) -> Vec<&[u8]> { - let trimmed = if bytes.last().copied() == Some(0) { - &bytes[..bytes.len() - 1] - } else { - bytes - }; + let trimmed = if bytes.last().copied() == Some(0) { &bytes[..bytes.len() - 1] } else { bytes }; if trimmed.is_empty() { return Vec::new(); } diff --git a/crates/plugin-wasm/src/config.rs b/crates/plugin-wasm/src/config.rs index 6c96b99f..a1ac0643 100644 --- a/crates/plugin-wasm/src/config.rs +++ b/crates/plugin-wasm/src/config.rs @@ -19,11 +19,53 @@ pub struct WasmLimits { pub fuel_per_call: Option, } +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct OciAuthConfig { + /// Optional registry hint, for example `registry.cn-hangzhou.aliyuncs.com`. + #[serde(default)] + pub registry: Option, + /// Basic-auth username used for registry token exchange or direct registry auth. + #[serde(default)] + pub username: Option, + /// Basic-auth password used for registry token exchange or direct registry auth. + #[serde(default)] + pub password: Option, + /// Pre-issued bearer token for registries that do not need a token challenge exchange. + #[serde(default)] + pub bearer_token: Option, + /// Docker config `identitytoken`; treated as a bearer token by the registry client. + #[serde(default)] + pub identity_token: Option, +} + +fn default_use_cache() -> bool { + true +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] pub struct WasmPluginShellConfig { - /// `file://`、`http(s)://` 或本地路径。 + /// `file://`、`http(s)://`、OCI 镜像 URL 或本地路径。 pub url: String, + /// Optional OCI registry auth. Usually populated from Higress `imagePullSecret`. + #[serde(default)] + pub oci_auth: Option, + /// 可选 SHA-256 校验值，支持裸 hex 或 `sha256:`。 + /// + /// 配置该字段后，host 会在编译前校验拉取到的 wasm 字节；字段变化也会自动让模块缓存失效。 + #[serde(default)] + pub sha256: Option, + /// 可选模块缓存键。 + /// + /// 默认按 `url` 加 `sha256` 复用编译产物；当远端同 URL 发布新版本且未配置 sha256 时， + /// 可以把这里设置成版本号/etag/digest 来强制重新拉取并编译。 + #[serde(default)] + pub module_cache_key: Option, + /// 是否复用进程内 wasm Module 缓存。 + /// + /// 默认开启；关闭后每次创建/更新插件实例都会重新拉取并编译，适合开发调试。 + #[serde(default = "default_use_cache")] + pub use_cache: bool, /// 传给 guest `proxy_on_configure` 的配置：可为 JSON 对象;序列化为 YAML 字节给 hai 系插件。 #[serde(default)] pub plugin_config: serde_json::Value, @@ -63,6 +105,10 @@ impl Default for WasmPluginShellConfig { fn default() -> Self { Self { url: String::new(), + oci_auth: None, + sha256: None, + module_cache_key: None, + use_cache: default_use_cache(), plugin_config: serde_json::Value::Null, fail_strategy: FailStrategy::FailOpen, clusters: HashMap::new(), @@ -84,9 +130,7 @@ impl WasmPluginShellConfig { if self.plugin_config.is_null() { return Vec::new(); } - serde_yaml::to_string(&self.plugin_config) - .unwrap_or_default() - .into_bytes() + serde_yaml::to_string(&self.plugin_config).unwrap_or_default().into_bytes() } /// 给定 guest 传来的 cluster 字符串，返回基础 URL（`http://host:port`）。 diff --git a/crates/plugin-wasm/src/fetch.rs b/crates/plugin-wasm/src/fetch.rs index 8fa3a37f..d3519046 100644 --- a/crates/plugin-wasm/src/fetch.rs +++ b/crates/plugin-wasm/src/fetch.rs @@ -1,19 +1,365 @@ //! 同步拉取 WASM 字节（在 `Plugin::create` 同步上下文中使用）。 //! -//! 支持：`file://...` 与裸文件系统路径；`http(s)://...` 暂未在 reqwest blocking 下启用， -//! 后续按 OCI 接入时一起做。 +//! 支持：`file://...`、裸文件系统路径、`http(s)://...` 与 OCI 镜像 URL。 +//! 网络拉取通过临时线程运行 async reqwest，避免在 `Plugin::create` 这条同步路径里嵌套 tokio runtime。 +use crate::config::OciAuthConfig; use crate::error::WasmHostError; +use reqwest::header::{ACCEPT, WWW_AUTHENTICATE}; +use serde::Deserialize; +use std::{collections::HashMap, time::Duration}; + +const OCI_MANIFEST_ACCEPT: &str = "application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.artifact.manifest.v1+json"; +const OCI_BLOB_ACCEPT: &str = "application/vnd.module.wasm.content.layer.v1+wasm, application/wasm, application/vnd.wasm.content.layer.v1+wasm, application/octet-stream"; + +fn fetch_http_wasm_bytes_sync(url: &str) -> Result, WasmHostError> { + let url = url.to_string(); + std::thread::Builder::new() + .name("spacegate-wasm-fetch".to_string()) + .spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().map_err(|e| WasmHostError::Fetch(format!("build fetch runtime: {e}")))?; + rt.block_on(async move { + let client = reqwest::Client::builder().timeout(Duration::from_secs(30)).build().map_err(|e| WasmHostError::Fetch(format!("build http client: {e}")))?; + let resp = client + .get(&url) + .send() + .await + .map_err(|e| WasmHostError::Fetch(format!("GET {url}: {e}")))? + .error_for_status() + .map_err(|e| WasmHostError::Fetch(format!("GET {url}: {e}")))?; + let bytes = resp.bytes().await.map_err(|e| WasmHostError::Fetch(format!("read {url} body: {e}")))?; + Ok(bytes.to_vec()) + }) + }) + .map_err(|e| WasmHostError::Fetch(format!("spawn fetch thread: {e}")))? + .join() + .map_err(|_| WasmHostError::Fetch("fetch thread panicked".to_string()))? +} + +fn fetch_oci_wasm_bytes_sync(url: &str, auth: Option) -> Result, WasmHostError> { + let url = url.to_string(); + std::thread::Builder::new() + .name("spacegate-wasm-oci-fetch".to_string()) + .spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().map_err(|e| WasmHostError::Fetch(format!("build OCI fetch runtime: {e}")))?; + rt.block_on(async move { + let reference = OciReference::parse(&url)?; + if let Some(auth_registry) = auth.as_ref().and_then(|a| a.registry.as_deref()).filter(|v| !v.trim().is_empty()) { + if !auth_registry.eq_ignore_ascii_case(&reference.registry) { + return Err(WasmHostError::Fetch(format!( + "OCI auth registry `{auth_registry}` does not match image registry `{}`", + reference.registry + ))); + } + } + let client = reqwest::Client::builder().timeout(Duration::from_secs(60)).build().map_err(|e| WasmHostError::Fetch(format!("build OCI client: {e}")))?; + fetch_oci_wasm_bytes(&client, &reference, auth.as_ref()).await + }) + }) + .map_err(|e| WasmHostError::Fetch(format!("spawn OCI fetch thread: {e}")))? + .join() + .map_err(|_| WasmHostError::Fetch("OCI fetch thread panicked".to_string()))? +} pub fn fetch_wasm_bytes_sync(url_or_path: &str) -> Result, WasmHostError> { + fetch_wasm_bytes_sync_with_auth(url_or_path, None) +} + +pub fn fetch_wasm_bytes_sync_with_auth(url_or_path: &str, oci_auth: Option<&OciAuthConfig>) -> Result, WasmHostError> { let trim = url_or_path.trim(); if let Some(rest) = trim.strip_prefix("file://") { return std::fs::read(rest).map_err(|e| WasmHostError::Fetch(format!("read file {rest}: {e}"))); } if trim.starts_with("http://") || trim.starts_with("https://") { - return Err(WasmHostError::Fetch( - "http(s)://wasm 拉取暂未启用：请使用 file:// 或裸路径".to_string(), - )); + return fetch_http_wasm_bytes_sync(trim); + } + if is_oci_url(trim) { + return fetch_oci_wasm_bytes_sync(trim, oci_auth.cloned()); } std::fs::read(trim).map_err(|e| WasmHostError::Fetch(format!("read path {trim}: {e}"))) } + +pub fn is_oci_url(url: &str) -> bool { + let lower = url.to_ascii_lowercase(); + lower.starts_with("oci://") || lower.starts_with("docker://") || lower.starts_with("image://") || lower.starts_with("oci+http://") +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct OciReference { + scheme: &'static str, + registry: String, + repository: String, + reference: String, +} + +impl OciReference { + fn parse(url: &str) -> Result { + let trim = url.trim(); + let (scheme, rest) = if let Some(rest) = trim.strip_prefix("oci+http://") { + ("http", rest) + } else if let Some(rest) = trim.strip_prefix("oci://") { + (default_oci_scheme(rest), rest) + } else if let Some(rest) = trim.strip_prefix("docker://") { + (default_oci_scheme(rest), rest) + } else if let Some(rest) = trim.strip_prefix("image://") { + (default_oci_scheme(rest), rest) + } else { + return Err(WasmHostError::Fetch(format!("unsupported OCI URL scheme: {trim}"))); + }; + + let Some((registry, image)) = rest.split_once('/') else { + return Err(WasmHostError::Fetch(format!("OCI URL must include registry and repository: {trim}"))); + }; + if registry.trim().is_empty() || image.trim().is_empty() { + return Err(WasmHostError::Fetch(format!("OCI URL must include registry and repository: {trim}"))); + } + + let (repository, reference) = if let Some((repository, digest)) = image.rsplit_once('@') { + (repository, digest) + } else if let Some((repository, tag)) = split_tag(image) { + (repository, tag) + } else { + (image, "latest") + }; + if repository.trim().is_empty() || reference.trim().is_empty() { + return Err(WasmHostError::Fetch(format!("OCI URL must include repository and tag/digest: {trim}"))); + } + + Ok(Self { + scheme, + registry: registry.to_string(), + repository: repository.to_string(), + reference: reference.to_string(), + }) + } + + fn manifest_url(&self, reference: &str) -> String { + format!("{}://{}/v2/{}/manifests/{}", self.scheme, self.registry, self.repository, reference) + } + + fn blob_url(&self, digest: &str) -> String { + format!("{}://{}/v2/{}/blobs/{}", self.scheme, self.registry, self.repository, digest) + } +} + +fn default_oci_scheme(rest: &str) -> &'static str { + let registry = rest.split('/').next().unwrap_or_default(); + if registry.starts_with("localhost") || registry.starts_with("127.0.0.1") || registry.starts_with("[::1]") { + "http" + } else { + "https" + } +} + +fn split_tag(image: &str) -> Option<(&str, &str)> { + let slash = image.rfind('/').map(|idx| idx + 1).unwrap_or(0); + let colon = image[slash..].rfind(':').map(|idx| slash + idx)?; + Some((&image[..colon], &image[colon + 1..])) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct OciManifest { + #[serde(default)] + media_type: Option, + #[serde(default)] + manifests: Vec, + #[serde(default)] + layers: Vec, + #[serde(default)] + blobs: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct OciDescriptor { + media_type: String, + digest: String, + #[serde(default)] + platform: Option, +} + +#[derive(Debug, Deserialize)] +struct OciPlatform { + #[serde(default)] + architecture: Option, + #[serde(default)] + os: Option, +} + +async fn fetch_oci_wasm_bytes(client: &reqwest::Client, reference: &OciReference, auth: Option<&OciAuthConfig>) -> Result, WasmHostError> { + let manifest = fetch_oci_manifest(client, reference, &reference.reference, auth).await?; + let manifest = if is_index_manifest(&manifest) { + let child = select_manifest_descriptor(&manifest.manifests)?; + fetch_oci_manifest(client, reference, &child.digest, auth).await? + } else { + manifest + }; + let layer = select_wasm_descriptor(&manifest)?; + registry_get_bytes(client, &reference.blob_url(&layer.digest), OCI_BLOB_ACCEPT, reference, auth).await +} + +async fn fetch_oci_manifest(client: &reqwest::Client, reference: &OciReference, manifest_ref: &str, auth: Option<&OciAuthConfig>) -> Result { + let url = reference.manifest_url(manifest_ref); + let bytes = registry_get_bytes(client, &url, OCI_MANIFEST_ACCEPT, reference, auth).await?; + serde_json::from_slice(&bytes).map_err(|e| WasmHostError::Fetch(format!("parse OCI manifest {url}: {e}"))) +} + +async fn registry_get_bytes(client: &reqwest::Client, url: &str, accept: &str, reference: &OciReference, auth: Option<&OciAuthConfig>) -> Result, WasmHostError> { + let send = |token: Option<&str>| { + let req = client.get(url).header(ACCEPT, accept); + apply_registry_auth(req, auth, token) + }; + + let resp = send(None).send().await.map_err(|e| WasmHostError::Fetch(format!("GET {url}: {e}")))?; + let resp = if resp.status() == reqwest::StatusCode::UNAUTHORIZED { + let challenge = resp.headers().get(WWW_AUTHENTICATE).and_then(|v| v.to_str().ok()).unwrap_or_default(); + let token = fetch_bearer_token(client, challenge, reference, auth).await?; + send(Some(&token)).send().await.map_err(|e| WasmHostError::Fetch(format!("GET {url}: {e}")))? + } else { + resp + }; + + let status = resp.status(); + if !status.is_success() { + return Err(WasmHostError::Fetch(format!("GET {url}: {status}"))); + } + let bytes = resp.bytes().await.map_err(|e| WasmHostError::Fetch(format!("read OCI response {url}: {e}")))?; + Ok(bytes.to_vec()) +} + +fn apply_registry_auth(req: reqwest::RequestBuilder, auth: Option<&OciAuthConfig>, bearer_token: Option<&str>) -> reqwest::RequestBuilder { + if let Some(token) = bearer_token { + return req.bearer_auth(token); + } + let Some(auth) = auth else { + return req; + }; + if let Some(token) = auth.bearer_token.as_deref().or(auth.identity_token.as_deref()).filter(|v| !v.trim().is_empty()) { + return req.bearer_auth(token); + } + if let Some(username) = auth.username.as_deref().filter(|v| !v.trim().is_empty()) { + return req.basic_auth(username, auth.password.clone()); + } + req +} + +async fn fetch_bearer_token(client: &reqwest::Client, challenge: &str, reference: &OciReference, auth: Option<&OciAuthConfig>) -> Result { + let params = + parse_bearer_challenge(challenge).ok_or_else(|| WasmHostError::Fetch(format!("registry {} requires auth but did not return a Bearer challenge", reference.registry)))?; + let realm = params.get("realm").filter(|v| !v.trim().is_empty()).ok_or_else(|| WasmHostError::Fetch("Bearer auth challenge missing realm".to_string()))?; + let mut url = reqwest::Url::parse(realm).map_err(|e| WasmHostError::Fetch(format!("parse Bearer token realm {realm}: {e}")))?; + { + let mut query = url.query_pairs_mut(); + if let Some(service) = params.get("service").filter(|v| !v.trim().is_empty()) { + query.append_pair("service", service); + } + let scope = params.get("scope").cloned().unwrap_or_else(|| format!("repository:{}:pull", reference.repository)); + query.append_pair("scope", &scope); + } + + let req = apply_registry_auth(client.get(url.clone()), auth, None); + let resp = req.send().await.map_err(|e| WasmHostError::Fetch(format!("GET OCI token {url}: {e}")))?; + let status = resp.status(); + if !status.is_success() { + return Err(WasmHostError::Fetch(format!("GET OCI token {url}: {status}"))); + } + let bytes = resp.bytes().await.map_err(|e| WasmHostError::Fetch(format!("read OCI token {url}: {e}")))?; + let token: OciTokenResponse = serde_json::from_slice(&bytes).map_err(|e| WasmHostError::Fetch(format!("parse OCI token response {url}: {e}")))?; + token.token.or(token.access_token).filter(|v| !v.trim().is_empty()).ok_or_else(|| WasmHostError::Fetch(format!("OCI token response {url} did not include token"))) +} + +#[derive(Debug, Deserialize)] +struct OciTokenResponse { + #[serde(default)] + token: Option, + #[serde(default)] + access_token: Option, +} + +fn parse_bearer_challenge(header: &str) -> Option> { + let rest = header.trim().strip_prefix("Bearer ")?; + let mut params = HashMap::new(); + for part in split_quoted_commas(rest) { + let Some((key, value)) = part.split_once('=') else { + continue; + }; + params.insert(key.trim().to_ascii_lowercase(), value.trim().trim_matches('"').to_string()); + } + Some(params) +} + +fn split_quoted_commas(value: &str) -> Vec<&str> { + let mut parts = Vec::new(); + let mut start = 0; + let mut in_quotes = false; + for (idx, ch) in value.char_indices() { + match ch { + '"' => in_quotes = !in_quotes, + ',' if !in_quotes => { + parts.push(value[start..idx].trim()); + start = idx + 1; + } + _ => {} + } + } + parts.push(value[start..].trim()); + parts +} + +fn is_index_manifest(manifest: &OciManifest) -> bool { + manifest.media_type.as_deref().map(|mt| mt.contains("image.index") || mt.contains("manifest.list")).unwrap_or(false) || !manifest.manifests.is_empty() +} + +fn select_manifest_descriptor(manifests: &[OciDescriptor]) -> Result<&OciDescriptor, WasmHostError> { + manifests + .iter() + .find(|m| m.platform.as_ref().map(|p| p.architecture.as_deref() == Some("wasm") || p.os.as_deref() == Some("wasi")).unwrap_or(false)) + .or_else(|| manifests.first()) + .ok_or_else(|| WasmHostError::Fetch("OCI image index does not contain manifests".to_string())) +} + +fn select_wasm_descriptor(manifest: &OciManifest) -> Result<&OciDescriptor, WasmHostError> { + let descriptors = manifest.layers.iter().chain(manifest.blobs.iter()).collect::>(); + descriptors + .iter() + .copied() + .find(|layer| is_wasm_media_type(&layer.media_type)) + .or_else(|| (descriptors.len() == 1).then(|| descriptors[0])) + .ok_or_else(|| WasmHostError::Fetch("OCI image does not contain a wasm layer".to_string())) +} + +fn is_wasm_media_type(media_type: &str) -> bool { + matches!( + media_type, + "application/vnd.module.wasm.content.layer.v1+wasm" | "application/vnd.wasm.content.layer.v1+wasm" | "application/wasm" + ) || media_type.contains("wasm") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_oci_reference_tag_digest_and_default_tag() { + assert_eq!( + OciReference::parse("oci://registry.example.com/ns/plugin:v1").unwrap(), + OciReference { + scheme: "https", + registry: "registry.example.com".to_string(), + repository: "ns/plugin".to_string(), + reference: "v1".to_string(), + } + ); + assert_eq!(OciReference::parse("docker://localhost:5000/plugin").unwrap().reference, "latest"); + assert_eq!(OciReference::parse("image://registry.example.com/ns/plugin@sha256:abc").unwrap().reference, "sha256:abc"); + } + + #[test] + fn parses_bearer_challenge() { + let parsed = parse_bearer_challenge(r#"Bearer realm="https://auth.example/token",service="registry.example",scope="repository:ns/plugin:pull""#).unwrap(); + assert_eq!(parsed["realm"], "https://auth.example/token"); + assert_eq!(parsed["service"], "registry.example"); + assert_eq!(parsed["scope"], "repository:ns/plugin:pull"); + } +} diff --git a/crates/plugin-wasm/src/host_fn.rs b/crates/plugin-wasm/src/host_fn.rs index 44d6e803..0c682baf 100644 --- a/crates/plugin-wasm/src/host_fn.rs +++ b/crates/plugin-wasm/src/host_fn.rs @@ -27,10 +27,7 @@ use crate::shared::{ /// 把所有 proxy-wasm v0.2.1 host fn 注册到 linker。 /// /// `dispatch_tx` 用于把异步 HTTP 调用结果发送给 Vm 状态机。 -pub fn register_all( - linker: &mut Linker, - dispatch_tx: tokio::sync::mpsc::UnboundedSender<(u32, HttpCallResult)>, -) -> Result<(), wasmtime::Error> { +pub fn register_all(linker: &mut Linker, dispatch_tx: tokio::sync::mpsc::UnboundedSender<(u32, HttpCallResult)>) -> Result<(), wasmtime::Error> { register_log(linker)?; register_clock_and_tick(linker)?; register_context_control(linker)?; @@ -52,46 +49,38 @@ pub fn register_all( // ───────────────────────────────────────────────────────── fn register_log(linker: &mut Linker) -> Result<(), wasmtime::Error> { - linker.func_wrap( - "env", - "proxy_log", - |mut caller: Caller<'_, HostState>, level: i32, msg_ptr: i32, msg_size: i32| -> i32 { - let mem = match MemoryHelper::from_caller(&mut caller) { - Ok(m) => m, - Err(_) => return Status::InvalidMemoryAccess.as_i32(), - }; - let Ok(msg) = mem.read_string_lossy(caller.as_context(), msg_ptr as u32, msg_size as u32) else { - return Status::InvalidMemoryAccess.as_i32(); - }; - let Some(lvl) = log_level_to_tracing(level) else { - return Status::BadArgument.as_i32(); - }; - match lvl { - tracing::Level::TRACE => tracing::trace!(target: "spacegate_plugin_wasm::guest", "{msg}"), - tracing::Level::DEBUG => tracing::debug!(target: "spacegate_plugin_wasm::guest", "{msg}"), - tracing::Level::INFO => tracing::info!(target: "spacegate_plugin_wasm::guest", "{msg}"), - tracing::Level::WARN => tracing::warn!(target: "spacegate_plugin_wasm::guest", "{msg}"), - tracing::Level::ERROR => tracing::error!(target: "spacegate_plugin_wasm::guest", "{msg}"), - } - Status::Ok.as_i32() - }, - )?; + linker.func_wrap("env", "proxy_log", |mut caller: Caller<'_, HostState>, level: i32, msg_ptr: i32, msg_size: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let Ok(msg) = mem.read_string_lossy(caller.as_context(), msg_ptr as u32, msg_size as u32) else { + return Status::InvalidMemoryAccess.as_i32(); + }; + let Some(lvl) = log_level_to_tracing(level) else { + return Status::BadArgument.as_i32(); + }; + match lvl { + tracing::Level::TRACE => tracing::trace!(target: "spacegate_plugin_wasm::guest", "{msg}"), + tracing::Level::DEBUG => tracing::debug!(target: "spacegate_plugin_wasm::guest", "{msg}"), + tracing::Level::INFO => tracing::info!(target: "spacegate_plugin_wasm::guest", "{msg}"), + tracing::Level::WARN => tracing::warn!(target: "spacegate_plugin_wasm::guest", "{msg}"), + tracing::Level::ERROR => tracing::error!(target: "spacegate_plugin_wasm::guest", "{msg}"), + } + Status::Ok.as_i32() + })?; - linker.func_wrap( - "env", - "proxy_get_log_level", - |mut caller: Caller<'_, HostState>, return_ptr: i32| -> i32 { - let mem = match MemoryHelper::from_caller(&mut caller) { - Ok(m) => m, - Err(_) => return Status::InvalidMemoryAccess.as_i32(), - }; - let lvl: LogLevel = host_max_log_level(); - if mem.write_u32(caller.as_context_mut(), return_ptr as u32, lvl.as_i32() as u32).is_err() { - return Status::InvalidMemoryAccess.as_i32(); - } - Status::Ok.as_i32() - }, - )?; + linker.func_wrap("env", "proxy_get_log_level", |mut caller: Caller<'_, HostState>, return_ptr: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let lvl: LogLevel = host_max_log_level(); + if mem.write_u32(caller.as_context_mut(), return_ptr as u32, lvl.as_i32() as u32).is_err() { + return Status::InvalidMemoryAccess.as_i32(); + } + Status::Ok.as_i32() + })?; Ok(()) } @@ -100,49 +89,37 @@ fn register_log(linker: &mut Linker) -> Result<(), wasmtime::Error> { // ───────────────────────────────────────────────────────── fn register_clock_and_tick(linker: &mut Linker) -> Result<(), wasmtime::Error> { - linker.func_wrap( - "env", - "proxy_get_current_time_nanoseconds", - |mut caller: Caller<'_, HostState>, return_ptr: i32| -> i32 { - let mem = match MemoryHelper::from_caller(&mut caller) { - Ok(m) => m, - Err(_) => return Status::InvalidMemoryAccess.as_i32(), - }; - let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos() as u64).unwrap_or(0); - if mem.write_u64(caller.as_context_mut(), return_ptr as u32, nanos).is_err() { - return Status::InvalidMemoryAccess.as_i32(); - } - Status::Ok.as_i32() - }, - )?; + linker.func_wrap("env", "proxy_get_current_time_nanoseconds", |mut caller: Caller<'_, HostState>, return_ptr: i32| -> i32 { + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos() as u64).unwrap_or(0); + if mem.write_u64(caller.as_context_mut(), return_ptr as u32, nanos).is_err() { + return Status::InvalidMemoryAccess.as_i32(); + } + Status::Ok.as_i32() + })?; - linker.func_wrap( - "env", - "proxy_set_tick_period_milliseconds", - |mut caller: Caller<'_, HostState>, period: i32| -> i32 { - caller.data_mut().tick_period_ms = if period > 0 { Some(period as u32) } else { None }; - Status::Ok.as_i32() - }, - )?; + linker.func_wrap("env", "proxy_set_tick_period_milliseconds", |mut caller: Caller<'_, HostState>, period: i32| -> i32 { + caller.data_mut().tick_period_ms = if period > 0 { Some(period as u32) } else { None }; + Status::Ok.as_i32() + })?; Ok(()) } fn register_context_control(linker: &mut Linker) -> Result<(), wasmtime::Error> { // proxy_set_effective_context(context_id) -> Status - linker.func_wrap( - "env", - "proxy_set_effective_context", - |mut caller: Caller<'_, HostState>, ctx_id: i32| -> i32 { - let cid = ctx_id as u32; - let st = caller.data_mut(); - if st.contexts.contains_key(&cid) || cid == st.root_context_id { - st.effective_context = cid; - Status::Ok.as_i32() - } else { - Status::BadArgument.as_i32() - } - }, - )?; + linker.func_wrap("env", "proxy_set_effective_context", |mut caller: Caller<'_, HostState>, ctx_id: i32| -> i32 { + let cid = ctx_id as u32; + let st = caller.data_mut(); + if st.contexts.contains_key(&cid) || cid == st.root_context_id { + st.effective_context = cid; + Status::Ok.as_i32() + } else { + Status::BadArgument.as_i32() + } + })?; // proxy_done() -> Status // @@ -175,39 +152,31 @@ fn register_stream_control(linker: &mut Linker) -> Result<(), wasmtim // 我们 host 端仅处理 HTTP_REQUEST/HTTP_RESPONSE 的 continue：把当前 ctx 的 // continue_requested 置 true，Vm 状态机据此退出 await loop。Downstream/Upstream // 我们不接 TCP 层 → 返回 UNIMPLEMENTED（spec 允许）。 - linker.func_wrap( - "env", - "proxy_continue_stream", - |mut caller: Caller<'_, HostState>, stream_type: i32| -> i32 { - let Some(st_kind) = StreamType::from_i32(stream_type) else { - return Status::BadArgument.as_i32(); - }; - match st_kind { - StreamType::HttpRequest | StreamType::HttpResponse => { - let st = caller.data(); - let ctx_id = st.effective_context; - if let Some(ctx) = caller.data_mut().contexts.get_mut(&ctx_id) { - ctx.continue_requested = true; - } - Status::Ok.as_i32() + linker.func_wrap("env", "proxy_continue_stream", |mut caller: Caller<'_, HostState>, stream_type: i32| -> i32 { + let Some(st_kind) = StreamType::from_i32(stream_type) else { + return Status::BadArgument.as_i32(); + }; + match st_kind { + StreamType::HttpRequest | StreamType::HttpResponse => { + let st = caller.data(); + let ctx_id = st.effective_context; + if let Some(ctx) = caller.data_mut().contexts.get_mut(&ctx_id) { + ctx.continue_requested = true; } - StreamType::Downstream | StreamType::Upstream => Status::Unimplemented.as_i32(), + Status::Ok.as_i32() } - }, - )?; + StreamType::Downstream | StreamType::Upstream => Status::Unimplemented.as_i32(), + } + })?; // proxy_close_stream(stream_type) -> Status - linker.func_wrap( - "env", - "proxy_close_stream", - |_caller: Caller<'_, HostState>, stream_type: i32| -> i32 { - match StreamType::from_i32(stream_type) { - Some(StreamType::HttpRequest) | Some(StreamType::HttpResponse) => Status::Ok.as_i32(), - Some(StreamType::Downstream) | Some(StreamType::Upstream) => Status::Unimplemented.as_i32(), - None => Status::BadArgument.as_i32(), - } - }, - )?; + linker.func_wrap("env", "proxy_close_stream", |_caller: Caller<'_, HostState>, stream_type: i32| -> i32 { + match StreamType::from_i32(stream_type) { + Some(StreamType::HttpRequest) | Some(StreamType::HttpResponse) => Status::Ok.as_i32(), + Some(StreamType::Downstream) | Some(StreamType::Upstream) => Status::Unimplemented.as_i32(), + None => Status::BadArgument.as_i32(), + } + })?; Ok(()) } @@ -232,13 +201,7 @@ fn register_buffer(linker: &mut Linker) -> Result<(), wasmtime::Error linker.func_wrap( "env", "proxy_get_buffer_bytes", - |mut caller: Caller<'_, HostState>, - buffer_type: i32, - start: i32, - max_size: i32, - return_data_ptr: i32, - return_size_ptr: i32| - -> i32 { + |mut caller: Caller<'_, HostState>, buffer_type: i32, start: i32, max_size: i32, return_data_ptr: i32, return_size_ptr: i32| -> i32 { let Some(buf_type) = BufferType::from_i32(buffer_type) else { return Status::BadArgument.as_i32(); }; @@ -291,13 +254,7 @@ fn register_buffer(linker: &mut Linker) -> Result<(), wasmtime::Error linker.func_wrap( "env", "proxy_set_buffer_bytes", - |mut caller: Caller<'_, HostState>, - buffer_type: i32, - start: i32, - size: i32, - data_ptr: i32, - data_size: i32| - -> i32 { + |mut caller: Caller<'_, HostState>, buffer_type: i32, start: i32, size: i32, data_ptr: i32, data_size: i32| -> i32 { let Some(buf_type) = BufferType::from_i32(buffer_type) else { return Status::BadArgument.as_i32(); }; @@ -430,13 +387,7 @@ fn register_headers(linker: &mut Linker) -> Result<(), wasmtime::Erro linker.func_wrap( "env", "proxy_get_header_map_value", - |mut caller: Caller<'_, HostState>, - map_type: i32, - key_ptr: i32, - key_size: i32, - return_data_ptr: i32, - return_size_ptr: i32| - -> i32 { + |mut caller: Caller<'_, HostState>, map_type: i32, key_ptr: i32, key_size: i32, return_data_ptr: i32, return_size_ptr: i32| -> i32 { let Some(mt) = MapType::from_i32(map_type) else { return Status::BadArgument.as_i32(); }; @@ -464,13 +415,7 @@ fn register_headers(linker: &mut Linker) -> Result<(), wasmtime::Erro linker.func_wrap( "env", "proxy_add_header_map_value", - |mut caller: Caller<'_, HostState>, - map_type: i32, - key_ptr: i32, - key_size: i32, - value_ptr: i32, - value_size: i32| - -> i32 { + |mut caller: Caller<'_, HostState>, map_type: i32, key_ptr: i32, key_size: i32, value_ptr: i32, value_size: i32| -> i32 { let Some(mt) = MapType::from_i32(map_type) else { return Status::BadArgument.as_i32(); }; @@ -494,13 +439,7 @@ fn register_headers(linker: &mut Linker) -> Result<(), wasmtime::Erro linker.func_wrap( "env", "proxy_replace_header_map_value", - |mut caller: Caller<'_, HostState>, - map_type: i32, - key_ptr: i32, - key_size: i32, - value_ptr: i32, - value_size: i32| - -> i32 { + |mut caller: Caller<'_, HostState>, map_type: i32, key_ptr: i32, key_size: i32, value_ptr: i32, value_size: i32| -> i32 { let Some(mt) = MapType::from_i32(map_type) else { return Status::BadArgument.as_i32(); }; @@ -692,13 +631,7 @@ fn register_http_call(linker: &mut Linker, dispatch_tx: tokio::sync:: return Status::BadArgument.as_i32(); } let st = caller.data(); - let base = st.shell_cfg.resolve_cluster(&cluster).or_else(|| { - if !authority.is_empty() { - Some(format!("http://{authority}")) - } else { - None - } - }); + let base = st.shell_cfg.resolve_cluster(&cluster).or_else(|| if !authority.is_empty() { Some(format!("http://{authority}")) } else { None }); let Some(base) = base else { warn!(target: "spacegate_plugin_wasm", cluster = %cluster, "dispatch_http_call: cluster not configured"); return Status::BadArgument.as_i32(); @@ -786,13 +719,7 @@ fn register_shared_data_and_queue(linker: &mut Linker) -> Result<(), linker.func_wrap( "env", "proxy_get_shared_data", - |mut caller: Caller<'_, HostState>, - k_ptr: i32, - k_size: i32, - v_data_ptr: i32, - v_size_ptr: i32, - cas_ptr: i32| - -> i32 { + |mut caller: Caller<'_, HostState>, k_ptr: i32, k_size: i32, v_data_ptr: i32, v_size_ptr: i32, cas_ptr: i32| -> i32 { let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, Err(_) => return Status::InvalidMemoryAccess.as_i32(), @@ -872,13 +799,7 @@ fn register_shared_data_and_queue(linker: &mut Linker) -> Result<(), linker.func_wrap( "env", "proxy_resolve_shared_queue", - |mut caller: Caller<'_, HostState>, - vid_ptr: i32, - vid_size: i32, - n_ptr: i32, - n_size: i32, - return_qid_ptr: i32| - -> i32 { + |mut caller: Caller<'_, HostState>, vid_ptr: i32, vid_size: i32, n_ptr: i32, n_size: i32, return_qid_ptr: i32| -> i32 { let mem = match MemoryHelper::from_caller(&mut caller) { Ok(m) => m, Err(_) => return Status::InvalidMemoryAccess.as_i32(), @@ -975,49 +896,37 @@ fn register_metrics(linker: &mut Linker) -> Result<(), wasmtime::Erro )?; // proxy_record_metric(mid, value: u64) -> Status - linker.func_wrap( - "env", - "proxy_record_metric", - |_caller: Caller<'_, HostState>, mid: i32, value: i64| -> i32 { - match metric_record(mid as u32, value as u64) { - MetricOpResult::Ok => Status::Ok.as_i32(), - MetricOpResult::NotFound => Status::NotFound.as_i32(), - MetricOpResult::BadArgument => Status::BadArgument.as_i32(), - } - }, - )?; + linker.func_wrap("env", "proxy_record_metric", |_caller: Caller<'_, HostState>, mid: i32, value: i64| -> i32 { + match metric_record(mid as u32, value as u64) { + MetricOpResult::Ok => Status::Ok.as_i32(), + MetricOpResult::NotFound => Status::NotFound.as_i32(), + MetricOpResult::BadArgument => Status::BadArgument.as_i32(), + } + })?; // proxy_increment_metric(mid, delta: i64) -> Status - linker.func_wrap( - "env", - "proxy_increment_metric", - |_caller: Caller<'_, HostState>, mid: i32, delta: i64| -> i32 { - match metric_increment(mid as u32, delta) { - MetricOpResult::Ok => Status::Ok.as_i32(), - MetricOpResult::NotFound => Status::NotFound.as_i32(), - MetricOpResult::BadArgument => Status::BadArgument.as_i32(), - } - }, - )?; + linker.func_wrap("env", "proxy_increment_metric", |_caller: Caller<'_, HostState>, mid: i32, delta: i64| -> i32 { + match metric_increment(mid as u32, delta) { + MetricOpResult::Ok => Status::Ok.as_i32(), + MetricOpResult::NotFound => Status::NotFound.as_i32(), + MetricOpResult::BadArgument => Status::BadArgument.as_i32(), + } + })?; // proxy_get_metric(mid, *return_value) -> Status - linker.func_wrap( - "env", - "proxy_get_metric", - |mut caller: Caller<'_, HostState>, mid: i32, return_ptr: i32| -> i32 { - let Some(v) = metric_get(mid as u32) else { - return Status::NotFound.as_i32(); - }; - let mem = match MemoryHelper::from_caller(&mut caller) { - Ok(m) => m, - Err(_) => return Status::InvalidMemoryAccess.as_i32(), - }; - if mem.write_u64(caller.as_context_mut(), return_ptr as u32, v).is_err() { - return Status::InvalidMemoryAccess.as_i32(); - } - Status::Ok.as_i32() - }, - )?; + linker.func_wrap("env", "proxy_get_metric", |mut caller: Caller<'_, HostState>, mid: i32, return_ptr: i32| -> i32 { + let Some(v) = metric_get(mid as u32) else { + return Status::NotFound.as_i32(); + }; + let mem = match MemoryHelper::from_caller(&mut caller) { + Ok(m) => m, + Err(_) => return Status::InvalidMemoryAccess.as_i32(), + }; + if mem.write_u64(caller.as_context_mut(), return_ptr as u32, v).is_err() { + return Status::InvalidMemoryAccess.as_i32(); + } + Status::Ok.as_i32() + })?; Ok(()) } @@ -1157,43 +1066,24 @@ fn register_grpc_unimplemented(linker: &mut Linker) -> Result<(), was linker.func_wrap( "env", "proxy_grpc_call", - |_caller: Caller<'_, HostState>, - _a: i32, - _b: i32, - _c: i32, - _d: i32, - _e: i32, - _f: i32, - _g: i32, - _h: i32, - _i: i32, - _j: i32, - _k: i32, - _l: i32| - -> i32 { Status::Unimplemented.as_i32() }, + |_caller: Caller<'_, HostState>, _a: i32, _b: i32, _c: i32, _d: i32, _e: i32, _f: i32, _g: i32, _h: i32, _i: i32, _j: i32, _k: i32, _l: i32| -> i32 { + Status::Unimplemented.as_i32() + }, )?; linker.func_wrap( "env", "proxy_grpc_stream", - |_caller: Caller<'_, HostState>, - _a: i32, - _b: i32, - _c: i32, - _d: i32, - _e: i32, - _f: i32, - _g: i32, - _h: i32, - _i: i32| - -> i32 { Status::Unimplemented.as_i32() }, - )?; - linker.func_wrap("env", "proxy_grpc_cancel", |_caller: Caller<'_, HostState>, _t: i32| -> i32 { Status::Unimplemented.as_i32() })?; - linker.func_wrap("env", "proxy_grpc_close", |_caller: Caller<'_, HostState>, _t: i32| -> i32 { Status::Unimplemented.as_i32() })?; - linker.func_wrap( - "env", - "proxy_grpc_send", - |_caller: Caller<'_, HostState>, _t: i32, _m: i32, _ms: i32, _eos: i32| -> i32 { Status::Unimplemented.as_i32() }, + |_caller: Caller<'_, HostState>, _a: i32, _b: i32, _c: i32, _d: i32, _e: i32, _f: i32, _g: i32, _h: i32, _i: i32| -> i32 { Status::Unimplemented.as_i32() }, )?; + linker.func_wrap("env", "proxy_grpc_cancel", |_caller: Caller<'_, HostState>, _t: i32| -> i32 { + Status::Unimplemented.as_i32() + })?; + linker.func_wrap("env", "proxy_grpc_close", |_caller: Caller<'_, HostState>, _t: i32| -> i32 { + Status::Unimplemented.as_i32() + })?; + linker.func_wrap("env", "proxy_grpc_send", |_caller: Caller<'_, HostState>, _t: i32, _m: i32, _ms: i32, _eos: i32| -> i32 { + Status::Unimplemented.as_i32() + })?; Ok(()) } diff --git a/crates/plugin-wasm/src/host_state.rs b/crates/plugin-wasm/src/host_state.rs index 889c3677..801acc72 100644 --- a/crates/plugin-wasm/src/host_state.rs +++ b/crates/plugin-wasm/src/host_state.rs @@ -148,10 +148,7 @@ pub struct HostState { impl HostState { pub fn new(shell_cfg: Arc) -> Self { let configuration = shell_cfg.configuration_bytes(); - let http_client = reqwest::Client::builder() - .pool_max_idle_per_host(8) - .build() - .unwrap_or_else(|_| reqwest::Client::new()); + let http_client = reqwest::Client::builder().pool_max_idle_per_host(8).build().unwrap_or_else(|_| reqwest::Client::new()); let plugin_name = shell_cfg.plugin_name.clone(); let plugin_root_id = shell_cfg.plugin_root_id.clone(); let plugin_vm_id = shell_cfg.plugin_vm_id.clone(); diff --git a/crates/plugin-wasm/src/runtime.rs b/crates/plugin-wasm/src/runtime.rs index 8001d78a..4b828412 100644 --- a/crates/plugin-wasm/src/runtime.rs +++ b/crates/plugin-wasm/src/runtime.rs @@ -4,11 +4,13 @@ use std::sync::Arc; use moka::sync::Cache; use once_cell::sync::OnceCell; +use sha2::{Digest, Sha256}; use wasmtime::Module; +use crate::config::WasmPluginShellConfig; use crate::engine::shared_engine; use crate::error::WasmHostError; -use crate::fetch::fetch_wasm_bytes_sync; +use crate::fetch::fetch_wasm_bytes_sync_with_auth; /// 进程内模块缓存（键：wasm `url` 字符串）。 pub struct WasmModuleCache { @@ -25,18 +27,48 @@ impl WasmModuleCache { } /// 拉取字节并编译；命中缓存则直接返回 `Arc`。 - pub fn get_or_compile(&self, url: &str) -> Result, WasmHostError> { - let key = url.to_string(); - if let Some(m) = self.inner.get(&key) { - return Ok(m); + pub fn get_or_compile(&self, cfg: &WasmPluginShellConfig) -> Result, WasmHostError> { + let key = module_cache_key(cfg); + if cfg.use_cache { + if let Some(m) = self.inner.get(&key) { + return Ok(m); + } } - let bytes = fetch_wasm_bytes_sync(url)?; + let bytes = fetch_wasm_bytes_sync_with_auth(cfg.url.trim(), cfg.oci_auth.as_ref())?; + verify_sha256(&bytes, cfg.sha256.as_deref())?; let m = Arc::new(Module::new(self.engine, &bytes)?); - self.inner.insert(key, m.clone()); + if cfg.use_cache { + self.inner.insert(key, m.clone()); + } Ok(m) } } +fn module_cache_key(cfg: &WasmPluginShellConfig) -> String { + let mut key = cfg.module_cache_key.as_deref().filter(|s| !s.trim().is_empty()).unwrap_or_else(|| cfg.url.trim()).to_string(); + if let Some(sha256) = cfg.sha256.as_deref().filter(|s| !s.trim().is_empty()) { + key.push_str("#sha256="); + key.push_str(normalize_sha256(sha256)); + } + key +} + +fn normalize_sha256(s: &str) -> &str { + s.trim().strip_prefix("sha256:").unwrap_or_else(|| s.trim()) +} + +fn verify_sha256(bytes: &[u8], expected: Option<&str>) -> Result<(), WasmHostError> { + let Some(expected) = expected.map(normalize_sha256).filter(|s| !s.is_empty()) else { + return Ok(()); + }; + let actual = format!("{:x}", Sha256::digest(bytes)); + if actual.eq_ignore_ascii_case(expected) { + Ok(()) + } else { + Err(WasmHostError::Fetch(format!("sha256 mismatch: expected {expected}, actual {actual}",))) + } +} + static CACHE: OnceCell = OnceCell::new(); /// 默认缓存（容量 64）；多实例同 URL 共享编译结果。 diff --git a/crates/plugin-wasm/src/shared.rs b/crates/plugin-wasm/src/shared.rs index 9dbd4c73..d30f23ba 100644 --- a/crates/plugin-wasm/src/shared.rs +++ b/crates/plugin-wasm/src/shared.rs @@ -172,7 +172,14 @@ pub fn metric_define(kind: MetricType, name: &str) -> u32 { } g.next_id = g.next_id.wrapping_add(1).max(1); let id = g.next_id; - g.by_id.insert(id, MetricEntry { kind, value: 0, name: name.to_string() }); + g.by_id.insert( + id, + MetricEntry { + kind, + value: 0, + name: name.to_string(), + }, + ); g.by_name.insert(name.to_string(), id); id } diff --git a/crates/plugin-wasm/src/shell.rs b/crates/plugin-wasm/src/shell.rs index 3251fffa..43428dec 100644 --- a/crates/plugin-wasm/src/shell.rs +++ b/crates/plugin-wasm/src/shell.rs @@ -89,7 +89,7 @@ impl Plugin for WasmPluginShell { "wasm plugin: create with config" ); let cache = default_module_cache(); - let module = cache.get_or_compile(cfg.url.trim()).map_err(|e| -> BoxError { format!("compile wasm: {e}").into() })?; + let module = cache.get_or_compile(&cfg).map_err(|e| -> BoxError { format!("compile wasm: {e}").into() })?; let cfg = Arc::new(cfg); let vm = Vm::new(&module, cfg.clone()).map_err(|e| -> BoxError { format!("Vm::new: {e}").into() })?; let vm = Arc::new(AsyncMutex::new(vm)); diff --git a/crates/plugin-wasm/src/vm.rs b/crates/plugin-wasm/src/vm.rs index d0e6ff46..84bbacbd 100644 --- a/crates/plugin-wasm/src/vm.rs +++ b/crates/plugin-wasm/src/vm.rs @@ -79,13 +79,9 @@ impl Vm { register_wasi_stubs(&mut linker)?; - let instance = linker - .instantiate(&mut store, module) - .map_err(|e| WasmHostError::Instantiate(format!("instantiate: {e}")))?; + let instance = linker.instantiate(&mut store, module).map_err(|e| WasmHostError::Instantiate(format!("instantiate: {e}")))?; - let memory = instance - .get_memory(&mut store, "memory") - .ok_or_else(|| WasmHostError::AbiViolation("no `memory` export".into()))?; + let memory = instance.get_memory(&mut store, "memory").ok_or_else(|| WasmHostError::AbiViolation("no `memory` export".into()))?; store.data_mut().memory = Some(memory); // spec §Memory management：优先 `proxy_on_memory_allocate`，否则回退 `malloc`。 if let Ok(alloc) = instance.get_typed_func::(&mut store, "proxy_on_memory_allocate") { @@ -93,9 +89,7 @@ impl Vm { } else if let Ok(alloc) = instance.get_typed_func::(&mut store, "malloc") { store.data_mut().alloc = Some(alloc); } else { - return Err(WasmHostError::AbiViolation( - "no memory allocator export (proxy_on_memory_allocate or malloc)".into(), - )); + return Err(WasmHostError::AbiViolation("no memory allocator export (proxy_on_memory_allocate or malloc)".into())); } // spec §Integration：先 `_initialize`；若不存在尝试 `_start`。 @@ -109,9 +103,8 @@ impl Vm { .get_typed_func::<(u32, u32), ()>(&mut store, "proxy_on_context_create") .map_err(|e| WasmHostError::AbiViolation(format!("get proxy_on_context_create: {e}")))?; let fn_on_vm_start = instance.get_typed_func::<(u32, u32), u32>(&mut store, "proxy_on_vm_start").ok(); - let fn_on_configure = instance - .get_typed_func::<(u32, u32), u32>(&mut store, "proxy_on_configure") - .map_err(|e| WasmHostError::AbiViolation(format!("get proxy_on_configure: {e}")))?; + let fn_on_configure = + instance.get_typed_func::<(u32, u32), u32>(&mut store, "proxy_on_configure").map_err(|e| WasmHostError::AbiViolation(format!("get proxy_on_configure: {e}")))?; let fn_on_request_headers = instance .get_typed_func::<(u32, u32, u32), u32>(&mut store, "proxy_on_request_headers") .map_err(|e| WasmHostError::AbiViolation(format!("get proxy_on_request_headers: {e}")))?; @@ -163,8 +156,7 @@ impl Vm { if let Some(ref f) = vm.fn_on_vm_start { vm.store.data_mut().effective_context = root_id; let cfg_len = vm.store.data().configuration.len() as u32; - let ok = f.call(&mut vm.store, (root_id, cfg_len)) - .map_err(|e| WasmHostError::GuestTrap { hook: "on_vm_start", source: e })?; + let ok = f.call(&mut vm.store, (root_id, cfg_len)).map_err(|e| WasmHostError::GuestTrap { hook: "on_vm_start", source: e })?; if ok == 0 { return Err(WasmHostError::Instantiate("guest on_vm_start returned 0 (=invalid VM configuration)".into())); } @@ -173,9 +165,7 @@ impl Vm { let cfg_len = vm.store.data().configuration.len() as u32; tracing::info!(target: "spacegate_plugin_wasm", cfg_len, "calling proxy_on_configure"); let configure_fn = vm.fn_on_configure.clone(); - let ok = configure_fn - .call(&mut vm.store, (root_id, cfg_len)) - .map_err(|e| WasmHostError::GuestTrap { hook: "on_configure", source: e })?; + let ok = configure_fn.call(&mut vm.store, (root_id, cfg_len)).map_err(|e| WasmHostError::GuestTrap { hook: "on_configure", source: e })?; if ok == 0 { warn!(target: "spacegate_plugin_wasm", "guest on_configure returned 0 (=invalid config)"); } @@ -185,8 +175,10 @@ impl Vm { fn create_context(&mut self, ctx_id: u32, parent_id: u32) -> Result<(), WasmHostError> { self.store.data_mut().effective_context = ctx_id; let f = self.fn_on_context_create.clone(); - f.call(&mut self.store, (ctx_id, parent_id)) - .map_err(|e| WasmHostError::GuestTrap { hook: "on_context_create", source: e })?; + f.call(&mut self.store, (ctx_id, parent_id)).map_err(|e| WasmHostError::GuestTrap { + hook: "on_context_create", + source: e, + })?; Ok(()) } @@ -206,9 +198,7 @@ impl Vm { let uri = parts.uri.clone(); let version = parts.version; let path = uri.path_and_query().map(|p| p.to_string()).unwrap_or_else(|| "/".to_string()); - let authority = uri.authority().map(|a| a.to_string()).unwrap_or_else(|| { - parts.headers.get(http::header::HOST).and_then(|h| h.to_str().ok()).unwrap_or("").to_string() - }); + let authority = uri.authority().map(|a| a.to_string()).unwrap_or_else(|| parts.headers.get(http::header::HOST).and_then(|h| h.to_str().ok()).unwrap_or("").to_string()); let scheme = uri.scheme_str().unwrap_or("http").to_string(); let headers = parts.headers.clone(); let pseudo = PseudoHeaders { @@ -239,9 +229,10 @@ impl Vm { let num_headers = (self.store.data().contexts[&http_ctx_id].request_headers.len() + 4) as u32; let end_of_stream_for_headers: u32 = if want_request_body { 0 } else { 1 }; let on_req_hdr = self.fn_on_request_headers.clone(); - let action_raw = on_req_hdr - .call(&mut self.store, (http_ctx_id, num_headers, end_of_stream_for_headers)) - .map_err(|e| WasmHostError::GuestTrap { hook: "on_request_headers", source: e })?; + let action_raw = on_req_hdr.call(&mut self.store, (http_ctx_id, num_headers, end_of_stream_for_headers)).map_err(|e| WasmHostError::GuestTrap { + hook: "on_request_headers", + source: e, + })?; let action = Action::from_u32(action_raw); debug!(target: "spacegate_plugin_wasm", http_ctx_id, ?action, "on_request_headers returned"); @@ -274,9 +265,10 @@ impl Vm { } } let on_req_body = self.fn_on_request_body.clone().expect("guarded by want_request_body"); - let action_raw = on_req_body - .call(&mut self.store, (http_ctx_id, body_size, 1)) - .map_err(|e| WasmHostError::GuestTrap { hook: "on_request_body", source: e })?; + let action_raw = on_req_body.call(&mut self.store, (http_ctx_id, body_size, 1)).map_err(|e| WasmHostError::GuestTrap { + hook: "on_request_body", + source: e, + })?; if Action::from_u32(action_raw) == Action::Pause { self.drive_until_continue(http_ctx_id).await?; } @@ -285,13 +277,7 @@ impl Vm { self.invoke_log_done_delete(http_ctx_id)?; return Ok(build_local_response(local)); } - let final_body = self - .store - .data() - .contexts - .get(&http_ctx_id) - .and_then(|c| c.request_body.clone()) - .unwrap_or(collected); + let final_body = self.store.data().contexts.get(&http_ctx_id).and_then(|c| c.request_body.clone()).unwrap_or(collected); (None, Some(final_body)) } else { (Some(body), None) @@ -304,9 +290,10 @@ impl Vm { ctx.stage = ContextStage::RequestTrailers; ctx.continue_requested = false; } - let action_raw = f - .call(&mut self.store, (http_ctx_id, 0)) - .map_err(|e| WasmHostError::GuestTrap { hook: "on_request_trailers", source: e })?; + let action_raw = f.call(&mut self.store, (http_ctx_id, 0)).map_err(|e| WasmHostError::GuestTrap { + hook: "on_request_trailers", + source: e, + })?; if Action::from_u32(action_raw) == Action::Pause { self.drive_until_continue(http_ctx_id).await?; } @@ -359,9 +346,10 @@ impl Vm { let want_response_body = self.fn_on_response_body.is_some(); let end_of_stream_for_resp_hdr: u32 = if want_response_body { 0 } else { 1 }; let on_resp_hdr = self.fn_on_response_headers.clone(); - let action_raw = on_resp_hdr - .call(&mut self.store, (http_ctx_id, (resp_headers.len() + 1) as u32, end_of_stream_for_resp_hdr)) - .map_err(|e| WasmHostError::GuestTrap { hook: "on_response_headers", source: e })?; + let action_raw = on_resp_hdr.call(&mut self.store, (http_ctx_id, (resp_headers.len() + 1) as u32, end_of_stream_for_resp_hdr)).map_err(|e| WasmHostError::GuestTrap { + hook: "on_response_headers", + source: e, + })?; if Action::from_u32(action_raw) == Action::Pause { self.drive_until_continue(http_ctx_id).await?; } @@ -388,9 +376,10 @@ impl Vm { st.effective_context = http_ctx_id; } } - let action_raw = f - .call(&mut self.store, (http_ctx_id, body_size, 1)) - .map_err(|e| WasmHostError::GuestTrap { hook: "on_response_body", source: e })?; + let action_raw = f.call(&mut self.store, (http_ctx_id, body_size, 1)).map_err(|e| WasmHostError::GuestTrap { + hook: "on_response_body", + source: e, + })?; if Action::from_u32(action_raw) == Action::Pause { self.drive_until_continue(http_ctx_id).await?; } @@ -408,9 +397,10 @@ impl Vm { ctx.stage = ContextStage::ResponseTrailers; ctx.continue_requested = false; } - let _ = f - .call(&mut self.store, (http_ctx_id, 0)) - .map_err(|e| WasmHostError::GuestTrap { hook: "on_response_trailers", source: e })?; + let _ = f.call(&mut self.store, (http_ctx_id, 0)).map_err(|e| WasmHostError::GuestTrap { + hook: "on_response_trailers", + source: e, + })?; // guest 可能改了 response_headers → 同步回 final_headers if let Some(ctx) = self.store.data().contexts.get(&http_ctx_id) { final_headers = ctx.response_headers.clone(); @@ -444,13 +434,7 @@ impl Vm { let Some((token, result)) = self.dispatch_rx.recv().await else { return Err(WasmHostError::Dispatch("dispatch channel closed".to_string())); }; - let source_ctx_id = self - .store - .data_mut() - .pending_calls - .remove(&token) - .map(|p| p.source_context_id) - .unwrap_or(ctx_id); + let source_ctx_id = self.store.data_mut().pending_calls.remove(&token).map(|p| p.source_context_id).unwrap_or(ctx_id); let header_count; let body_len; { @@ -468,8 +452,10 @@ impl Vm { } debug!(target: "spacegate_plugin_wasm", token, source_ctx_id, status = result.status, body_len, "fire proxy_on_http_call_response"); let f = self.fn_on_http_call_response.clone(); - f.call(&mut self.store, (source_ctx_id, token, header_count, body_len, 0)) - .map_err(|e| WasmHostError::GuestTrap { hook: "on_http_call_response", source: e })?; + f.call(&mut self.store, (source_ctx_id, token, header_count, body_len, 0)).map_err(|e| WasmHostError::GuestTrap { + hook: "on_http_call_response", + source: e, + })?; } } @@ -490,14 +476,7 @@ impl Vm { ctx.awaiting_done = true; } let v = f.call(&mut self.store, ctx_id).unwrap_or(1); - let done = v != 0 - || self - .store - .data() - .contexts - .get(&ctx_id) - .map(|c| c.done_marker) - .unwrap_or(true); + let done = v != 0 || self.store.data().contexts.get(&ctx_id).map(|c| c.done_marker).unwrap_or(true); if !done { warn!( target: "spacegate_plugin_wasm", @@ -526,10 +505,11 @@ impl Vm { /// /// 失败要么是 guest trap（要么后台任务自停），要么是 guest 没导出 `proxy_on_tick`——后者直接 Ok。 pub fn tick(&mut self) -> Result<(), WasmHostError> { - let Some(f) = self.fn_on_tick.clone() else { return Ok(()); }; + let Some(f) = self.fn_on_tick.clone() else { + return Ok(()); + }; self.store.data_mut().effective_context = self.root_id; - f.call(&mut self.store, self.root_id) - .map_err(|e| WasmHostError::GuestTrap { hook: "on_tick", source: e })?; + f.call(&mut self.store, self.root_id).map_err(|e| WasmHostError::GuestTrap { hook: "on_tick", source: e })?; Ok(()) } } @@ -610,7 +590,9 @@ pub fn register_wasi_stubs(linker: &mut Linker) -> Result<(), wasmtim wasi_errno::SUCCESS }, )?; - linker.func_wrap("wasi_snapshot_preview1", "environ_get", |_c: wasmtime::Caller<'_, HostState>, _a: i32, _b: i32| -> i32 { wasi_errno::SUCCESS })?; + linker.func_wrap("wasi_snapshot_preview1", "environ_get", |_c: wasmtime::Caller<'_, HostState>, _a: i32, _b: i32| -> i32 { + wasi_errno::SUCCESS + })?; linker.func_wrap( "wasi_snapshot_preview1", "environ_sizes_get", @@ -628,7 +610,9 @@ pub fn register_wasi_stubs(linker: &mut Linker) -> Result<(), wasmtim wasi_errno::SUCCESS }, )?; - linker.func_wrap("wasi_snapshot_preview1", "args_get", |_c: wasmtime::Caller<'_, HostState>, _a: i32, _b: i32| -> i32 { wasi_errno::SUCCESS })?; + linker.func_wrap("wasi_snapshot_preview1", "args_get", |_c: wasmtime::Caller<'_, HostState>, _a: i32, _b: i32| -> i32 { + wasi_errno::SUCCESS + })?; linker.func_wrap( "wasi_snapshot_preview1", "args_sizes_get", diff --git a/crates/plugin-wasm/tests/http_call.rs b/crates/plugin-wasm/tests/http_call.rs index d3a21588..d54fef12 100644 --- a/crates/plugin-wasm/tests/http_call.rs +++ b/crates/plugin-wasm/tests/http_call.rs @@ -90,15 +90,10 @@ async fn start_mock_server(body_byte: u8) -> SocketAddr { tokio::spawn(async move { let svc = service_fn(move |_req: HyperRequest| async move { let body = Bytes::from(vec![body_byte]); - let resp = Response::builder() - .status(200) - .body(Full::new(body)) - .expect("build resp"); + let resp = Response::builder().status(200).body(Full::new(body)).expect("build resp"); Ok::<_, Infallible>(resp) }); - let _ = http1::Builder::new() - .serve_connection(TokioIo::new(stream), svc) - .await; + let _ = http1::Builder::new().serve_connection(TokioIo::new(stream), svc).await; }); } }); diff --git a/crates/plugin-wasm/tests/on_tick.rs b/crates/plugin-wasm/tests/on_tick.rs index 0597c340..a3533bd2 100644 --- a/crates/plugin-wasm/tests/on_tick.rs +++ b/crates/plugin-wasm/tests/on_tick.rs @@ -9,10 +9,10 @@ use std::path::PathBuf; use std::time::Duration; +use spacegate_model::{PluginInstanceId, PluginInstanceName}; use spacegate_plugin::{Plugin, PluginConfig}; use spacegate_plugin_wasm::shared::{shared_data_get, shared_data_set}; use spacegate_plugin_wasm::WasmPluginShell; -use spacegate_model::{PluginInstanceId, PluginInstanceName}; // ───────────────────────────────────────────────────────── // guest .wasm 定位/构建 diff --git a/crates/plugin-wasm/tests/runtime_fetch.rs b/crates/plugin-wasm/tests/runtime_fetch.rs new file mode 100644 index 00000000..2a445a53 --- /dev/null +++ b/crates/plugin-wasm/tests/runtime_fetch.rs @@ -0,0 +1,192 @@ +//! Covers wasm module loading concerns that sit below Proxy-Wasm execution: +//! remote fetch, digest verification, and cache invalidation. + +use std::collections::HashMap; +use std::convert::Infallible; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; + +use bytes::Bytes; +use http_body_util::Full; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use sha2::{Digest, Sha256}; +use spacegate_plugin_wasm::config::WasmPluginShellConfig; +use spacegate_plugin_wasm::fetch::fetch_wasm_bytes_sync; +use spacegate_plugin_wasm::runtime::WasmModuleCache; +use tokio::net::TcpListener; + +async fn start_bytes_server(body: Bytes) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("local_addr"); + tokio::spawn(async move { + loop { + let (stream, _) = match listener.accept().await { + Ok(s) => s, + Err(_) => return, + }; + let body = body.clone(); + tokio::spawn(async move { + let svc = service_fn(move |_req: Request| { + let body = body.clone(); + async move { Ok::<_, Infallible>(Response::new(Full::new(body))) } + }); + let _ = http1::Builder::new().serve_connection(TokioIo::new(stream), svc).await; + }); + } + }); + addr +} + +async fn start_oci_registry_server(wasm: Bytes) -> SocketAddr { + let digest = sha256_hex(&wasm); + let manifest = Bytes::from(format!( + r#"{{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": {{"mediaType": "application/vnd.unknown.config.v1+json", "digest": "sha256:{}", "size": 2}}, + "layers": [ + {{"mediaType": "application/vnd.module.wasm.content.layer.v1+wasm", "digest": "sha256:{digest}", "size": {}}} + ] +}}"#, + "0".repeat(64), + wasm.len() + )); + let mut routes = HashMap::new(); + routes.insert("/v2/plugin/manifests/v1".to_string(), manifest); + routes.insert(format!("/v2/plugin/blobs/sha256:{digest}"), wasm); + let routes = Arc::new(routes); + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("local_addr"); + tokio::spawn(async move { + loop { + let (stream, _) = match listener.accept().await { + Ok(s) => s, + Err(_) => return, + }; + let routes = routes.clone(); + tokio::spawn(async move { + let svc = service_fn(move |req: Request| { + let routes = routes.clone(); + async move { + let path = req.uri().path().to_string(); + if let Some(body) = routes.get(&path) { + Ok::<_, Infallible>(Response::new(Full::new(body.clone()))) + } else { + let mut resp = Response::new(Full::new(Bytes::from_static(b"not found"))); + *resp.status_mut() = StatusCode::NOT_FOUND; + Ok(resp) + } + } + }); + let _ = http1::Builder::new().serve_connection(TokioIo::new(stream), svc).await; + }); + } + }); + addr +} + +fn guest_manifest_path() -> PathBuf { + let mut p = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + p.push("tests"); + p.push("on_tick_guest"); + p.push("Cargo.toml"); + p +} + +fn guest_wasm_path() -> PathBuf { + let manifest = guest_manifest_path(); + let out = std::process::Command::new(env!("CARGO")) + .args(["metadata", "--no-deps", "--format-version", "1", "--manifest-path"]) + .arg(&manifest) + .output() + .expect("cargo metadata: spawn"); + assert!(out.status.success(), "cargo metadata failed: {}", String::from_utf8_lossy(&out.stderr)); + let meta: serde_json::Value = serde_json::from_slice(&out.stdout).expect("parse cargo metadata json"); + let target_dir = meta["target_directory"].as_str().expect("target_directory missing"); + PathBuf::from(target_dir).join("wasm32-wasip1").join("release").join("on_tick_guest.wasm") +} + +fn ensure_guest_built() -> PathBuf { + let wasm = guest_wasm_path(); + if !wasm.exists() { + let status = std::process::Command::new(env!("CARGO")) + .args(["build", "--release", "--target", "wasm32-wasip1", "--manifest-path"]) + .arg(guest_manifest_path()) + .status() + .expect("cargo build: spawn"); + assert!(status.success(), "on_tick_guest build failed"); + assert!(wasm.exists(), "wasm still missing after build: {wasm:?}"); + } + wasm +} + +fn sha256_hex(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +#[tokio::test] +async fn fetch_wasm_bytes_supports_http_urls() { + let expected = Bytes::from_static(b"hello wasm over http"); + let addr = start_bytes_server(expected.clone()).await; + let url = format!("http://{addr}/plugin.wasm"); + + let fetched = tokio::task::spawn_blocking(move || fetch_wasm_bytes_sync(&url)).await.expect("join").expect("fetch"); + + assert_eq!(fetched, expected); +} + +#[tokio::test] +async fn fetch_wasm_bytes_supports_oci_image_layers() { + let expected = Bytes::from_static(b"\0asm\x01\0\0\0"); + let addr = start_oci_registry_server(expected.clone()).await; + let url = format!("oci://{addr}/plugin:v1"); + + let fetched = tokio::task::spawn_blocking(move || fetch_wasm_bytes_sync(&url)).await.expect("join").expect("fetch"); + + assert_eq!(fetched, expected); +} + +#[test] +fn wasm_module_cache_uses_module_cache_key_for_invalidation() { + let wasm = ensure_guest_built(); + let bytes = std::fs::read(&wasm).expect("read wasm"); + let sha256 = sha256_hex(&bytes); + let cache = WasmModuleCache::new(8); + + let cfg_v1 = WasmPluginShellConfig { + url: format!("file://{}", wasm.display()), + sha256: Some(sha256.clone()), + module_cache_key: Some("on-tick:v1".to_string()), + ..Default::default() + }; + let first = cache.get_or_compile(&cfg_v1).expect("compile v1"); + let cached = cache.get_or_compile(&cfg_v1).expect("compile v1 cached"); + assert!(Arc::ptr_eq(&first, &cached)); + + let cfg_v2 = WasmPluginShellConfig { + module_cache_key: Some("on-tick:v2".to_string()), + ..cfg_v1 + }; + let second = cache.get_or_compile(&cfg_v2).expect("compile v2"); + assert!(!Arc::ptr_eq(&first, &second)); +} + +#[test] +fn wasm_module_cache_rejects_sha256_mismatch() { + let wasm = ensure_guest_built(); + let cache = WasmModuleCache::new(8); + let cfg = WasmPluginShellConfig { + url: format!("file://{}", wasm.display()), + sha256: Some("sha256:0000000000000000000000000000000000000000000000000000000000000000".to_string()), + use_cache: false, + ..Default::default() + }; + + let err = cache.get_or_compile(&cfg).expect_err("expected sha mismatch"); + assert!(err.to_string().contains("sha256 mismatch"), "{err}"); +} diff --git a/crates/plugin-wasm/tests/sdk_examples.rs b/crates/plugin-wasm/tests/sdk_examples.rs index 4c21be5c..023960c4 100644 --- a/crates/plugin-wasm/tests/sdk_examples.rs +++ b/crates/plugin-wasm/tests/sdk_examples.rs @@ -18,9 +18,9 @@ use std::sync::Arc; use bytes::Bytes; use http_body_util::BodyExt; use hyper::service::service_fn; +use hyper::Request as HyperRequest; use spacegate_kernel::backend_service::ArcHyperService; use spacegate_kernel::helper_layers::function::Inner; -use hyper::Request as HyperRequest; use spacegate_kernel::{SgBody, SgRequest, SgResponse}; use spacegate_plugin_wasm::config::WasmPluginShellConfig; use spacegate_plugin_wasm::engine::shared_engine; @@ -133,12 +133,7 @@ async fn sdk_example_http_headers_hello() { let cfg = make_cfg(serde_json::json!({"mode": "headers"})); let mut vm = Vm::new(&module, cfg).expect("Vm::new"); - let req = HyperRequest::builder() - .method("GET") - .uri("http://example.test/hello") - .header("host", "example.test") - .body(SgBody::empty()) - .expect("build req"); + let req = HyperRequest::builder().method("GET").uri("http://example.test/hello").header("host", "example.test").body(SgBody::empty()).expect("build req"); let captured = CaptureState::default(); let inner = make_inner(captured.clone()); let resp = vm.process(req, inner).await.expect("process"); @@ -147,10 +142,7 @@ async fn sdk_example_http_headers_hello() { assert_eq!(resp.status(), 200); assert_eq!(body, Bytes::from_static(b"Hello, World!\n")); assert_eq!(resp.headers().get("hello").and_then(|v| v.to_str().ok()), Some("world")); - assert_eq!( - resp.headers().get("powered-by").and_then(|v| v.to_str().ok()), - Some("proxy-wasm") - ); + assert_eq!(resp.headers().get("powered-by").and_then(|v| v.to_str().ok()), Some("proxy-wasm")); assert!( !captured.invoked.load(std::sync::atomic::Ordering::SeqCst), "inner.call must NOT be invoked for local response" @@ -182,10 +174,7 @@ async fn sdk_example_http_headers_passthrough() { "on_response_headers should inject x-sdk-headers" ); // echo header 应该原路回来 - assert_eq!( - resp.headers().get("x-echo-foo").and_then(|v| v.to_str().ok()), - Some("bar") - ); + assert_eq!(resp.headers().get("x-echo-foo").and_then(|v| v.to_str().ok()), Some("bar")); } // ───────────────────────────────────────────────────────── @@ -210,10 +199,7 @@ async fn sdk_example_http_body_reverses_request_body() { assert_eq!(resp.status(), 200); // Inner 收到的应是反转后的字节，echo 回来后响应体也是它。 - assert_eq!( - captured.inbound_body.lock().await.clone().expect("body captured"), - Bytes::from_static(b"321-cba") - ); + assert_eq!(captured.inbound_body.lock().await.clone().expect("body captured"), Bytes::from_static(b"321-cba")); assert_eq!(body, Bytes::from_static(b"321-cba")); } @@ -227,12 +213,7 @@ async fn sdk_example_http_config_missing_header_rejected() { let cfg = make_cfg(serde_json::json!({"mode": "config", "required_header": "x-token"})); let mut vm = Vm::new(&module, cfg).expect("Vm::new"); - let req = HyperRequest::builder() - .method("GET") - .uri("http://example.test/") - .header("host", "example.test") - .body(SgBody::empty()) - .expect("build req"); + let req = HyperRequest::builder().method("GET").uri("http://example.test/").header("host", "example.test").body(SgBody::empty()).expect("build req"); let captured = CaptureState::default(); let resp = vm.process(req, make_inner(captured.clone())).await.expect("process"); let (resp, body) = full_body(resp).await; diff --git a/crates/plugin-wasm/tests/spec_compliance.rs b/crates/plugin-wasm/tests/spec_compliance.rs index 13f6fabe..c9491de8 100644 --- a/crates/plugin-wasm/tests/spec_compliance.rs +++ b/crates/plugin-wasm/tests/spec_compliance.rs @@ -137,10 +137,7 @@ impl GuestVm { } fn run_test(&mut self, scenario: u32) -> u32 { - let f: TypedFunc = self - .instance - .get_typed_func(&mut self.store, "__run_test") - .expect("__run_test export"); + let f: TypedFunc = self.instance.get_typed_func(&mut self.store, "__run_test").expect("__run_test export"); f.call(&mut self.store, scenario).expect("__run_test trap-free") } @@ -217,12 +214,7 @@ fn proxy_wasm_spec_v0_2_1_compliance() { let lr = ctx.local_response.as_ref().expect("local_response written by guest"); assert_eq!(lr.status, 418, "local_response.status"); assert_eq!(lr.body, Bytes::from_static(b"local body"), "local_response.body"); - let x_spec = lr - .headers - .get("x-spec") - .expect("x-spec header present") - .to_str() - .unwrap_or(""); + let x_spec = lr.headers.get("x-spec").expect("x-spec header present").to_str().unwrap_or(""); assert_eq!(x_spec, "teapot"); // (20) set_tick_period 应写入 HostState.tick_period_ms diff --git a/docs/k8s/gateway-api-compatibility.md b/docs/k8s/gateway-api-compatibility.md index 0985831b..617e7f66 100644 --- a/docs/k8s/gateway-api-compatibility.md +++ b/docs/k8s/gateway-api-compatibility.md @@ -143,3 +143,30 @@ Fields: - kind - `Gateway` `HTTPRoute` - namespace (option) - name + +### Higress-compatible WasmPlugin + +Spacegate can read Higress-style `extensions.higress.io/v1alpha1` `WasmPlugin` resources and translate them into the internal `code = "wasm"` plugin runtime configuration. + +Supported fields: + +- `spec.url` - local path, `file://`, `http://`, `https://`, or OCI wasm image URL (`oci://`, `docker://`, `image://`). +- `spec.sha256` - optional wasm byte digest, plain hex or `sha256:`. +- `spec.pluginName` - exposed to the proxy-wasm guest as `plugin_name`. +- `spec.defaultConfig` - converted to a Spacegate wasm plugin instance and mounted at Gateway level. +- `spec.defaultConfigDisable` - disables the generated Gateway-level default plugin instance. +- `spec.matchRules[].ingress` - generates per-rule wasm plugin instances and mounts them on matching Spacegate routes. +- `spec.matchRules[].domain` - generates per-rule wasm plugin instances and mounts them on routes whose hostnames match. +- `spec.matchRules[].service` - generates per-rule wasm plugin instances and mounts them on matching backends. +- `spec.matchRules[].config/configDisable` - configures or disables each generated rule-level plugin instance. +- `spec.failStrategy` - accepts `FAIL_OPEN`/`FAIL_CLOSE` and maps to Spacegate `fail_open`/`fail_close`. +- `spec.phase` - participates in plugin ordering (`AUTHN` before `AUTHZ` before unspecified before `STATS`). +- `spec.priority` - used inside the same phase; higher priority plugins are mounted earlier. +- `spec.imagePullPolicy` - `Always` disables the Spacegate module cache for that plugin instance. +- `spec.imagePullSecret` - for OCI URLs, Spacegate reads Docker config (`.dockerconfigjson`/`.dockercfg`) or basic-auth (`username`/`password`) Kubernetes Secrets and passes the registry credentials to the wasm runtime. +- `status` - Spacegate writes `observedGeneration`, `phase`, `digest`, and `message` during K8s watch reconciliation. + +Current limitations: + +- OCI layer selection supports wasm media types (`application/vnd.module.wasm.content.layer.v1+wasm`, `application/vnd.wasm.content.layer.v1+wasm`, `application/wasm`) and falls back to a single-layer artifact. +- `phase` currently maps to ordering only, not to separate Spacegate execution pipelines. diff --git a/examples/wasm-hello/README.md b/examples/wasm-hello/README.md index 264e5f2a..67312486 100644 --- a/examples/wasm-hello/README.md +++ b/examples/wasm-hello/README.md @@ -11,6 +11,24 @@ cd ../.. cp examples/wasm-hello/target/wasm32-wasip1/release/spacegate_wasm_hello.wasm resource/wasm/spacegate_wasm_hello.wasm ``` +If you rebuild the wasm, update `resource/wasm-hello-demo/plugin/wasm.hello-world.json` +with the new digest: + +```bash +shasum -a 256 resource/wasm/spacegate_wasm_hello.wasm +``` + +The wasm host also supports remote loading: + +```json +{ + "url": "https://example.com/plugins/spacegate_wasm_hello.wasm", + "sha256": "sha256:<64-char-hex-digest>", + "module_cache_key": "spacegate-wasm-hello:v1", + "use_cache": true +} +``` + Run Spacegate with the demo config from the repository root: ```bash diff --git a/resource/kube-manifests/higress-wasmplugin-crd.yaml b/resource/kube-manifests/higress-wasmplugin-crd.yaml new file mode 100644 index 00000000..acd09c3d --- /dev/null +++ b/resource/kube-manifests/higress-wasmplugin-crd.yaml @@ -0,0 +1,81 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: wasmplugins.extensions.higress.io +spec: + group: extensions.higress.io + scope: Namespaced + names: + plural: wasmplugins + singular: wasmplugin + kind: WasmPlugin + listKind: WasmPluginList + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + required: + - url + properties: + url: + type: string + pluginName: + type: string + sha256: + type: string + phase: + type: string + priority: + type: integer + format: int32 + imagePullPolicy: + type: string + imagePullSecret: + type: string + defaultConfigDisable: + type: boolean + failStrategy: + type: string + defaultConfig: + x-kubernetes-preserve-unknown-fields: true + matchRules: + type: array + items: + type: object + properties: + ingress: + type: array + items: + type: string + domain: + type: array + items: + type: string + service: + type: array + items: + type: string + configDisable: + type: boolean + config: + x-kubernetes-preserve-unknown-fields: true + status: + type: object + properties: + observedGeneration: + type: integer + format: int64 + phase: + type: string + digest: + type: string + message: + type: string diff --git a/resource/kube-manifests/spacegate-admin-server.yaml b/resource/kube-manifests/spacegate-admin-server.yaml index 3666ae20..c106c180 100644 --- a/resource/kube-manifests/spacegate-admin-server.yaml +++ b/resource/kube-manifests/spacegate-admin-server.yaml @@ -83,6 +83,26 @@ rules: - list - watch - delete + - apiGroups: + - extensions.higress.io + resources: + - wasmplugins + verbs: + - get + - create + - update + - patch + - list + - watch + - delete + - apiGroups: + - extensions.higress.io + resources: + - wasmplugins/status + verbs: + - get + - update + - patch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -122,4 +142,4 @@ spec: hostPort: 9080 env: - name: CONFIG - value: k8s:spacegate \ No newline at end of file + value: k8s:spacegate diff --git a/resource/kube-manifests/spacegate-gateway.yaml b/resource/kube-manifests/spacegate-gateway.yaml index f0bab316..cac7e4c5 100644 --- a/resource/kube-manifests/spacegate-gateway.yaml +++ b/resource/kube-manifests/spacegate-gateway.yaml @@ -126,6 +126,21 @@ rules: - get - list - watch + - apiGroups: + - extensions.higress.io + resources: + - wasmplugins + verbs: + - get + - list + - watch + - apiGroups: + - extensions.higress.io + resources: + - wasmplugins/status + verbs: + - get + - update --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/resource/kube-manifests/wasmplugin-hello-example.yaml b/resource/kube-manifests/wasmplugin-hello-example.yaml new file mode 100644 index 00000000..5e3ffeb9 --- /dev/null +++ b/resource/kube-manifests/wasmplugin-hello-example.yaml @@ -0,0 +1,19 @@ +apiVersion: extensions.higress.io/v1alpha1 +kind: WasmPlugin +metadata: + name: hello-world + namespace: spacegate +spec: + url: https://example.com/plugins/spacegate_wasm_hello.wasm + sha256: sha256:6b9dacbcbf5a2d9de9795737aeecd434dc6b261476486803419e9d62084e651c + pluginName: hello-world + phase: AUTHN + priority: 100 + failStrategy: FAIL_CLOSE + defaultConfig: + message: hello world + matchRules: + - domain: + - api.example.com + config: + message: hello api diff --git a/resource/wasm-hello-demo/plugin/wasm.hello-world.json b/resource/wasm-hello-demo/plugin/wasm.hello-world.json index b2327fb7..392941ee 100644 --- a/resource/wasm-hello-demo/plugin/wasm.hello-world.json +++ b/resource/wasm-hello-demo/plugin/wasm.hello-world.json @@ -1,5 +1,8 @@ { "url": "resource/wasm/spacegate_wasm_hello.wasm", + "sha256": "sha256:6b9dacbcbf5a2d9de9795737aeecd434dc6b261476486803419e9d62084e651c", + "module_cache_key": "spacegate-wasm-hello:v1", + "use_cache": true, "validate_on_create": false, "fail_strategy": "fail_close", "plugin_name": "hello-world", From 17c4c044a8d731573466a169ca16df33a5225d3b Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Fri, 15 May 2026 15:43:07 +0800 Subject: [PATCH 05/19] Add wasm plugin development workspace --- plugins/wasm/.cargo/config.toml | 3 ++ plugins/wasm/.gitignore | 4 ++ plugins/wasm/Cargo.toml | 21 ++++++++++ plugins/wasm/README.md | 60 ++++++++++++++++++++++++++++ plugins/wasm/hello-world/Cargo.toml | 13 ++++++ plugins/wasm/hello-world/plugin.yaml | 13 ++++++ plugins/wasm/hello-world/src/lib.rs | 50 +++++++++++++++++++++++ 7 files changed, 164 insertions(+) create mode 100644 plugins/wasm/.cargo/config.toml create mode 100644 plugins/wasm/.gitignore create mode 100644 plugins/wasm/Cargo.toml create mode 100644 plugins/wasm/README.md create mode 100644 plugins/wasm/hello-world/Cargo.toml create mode 100644 plugins/wasm/hello-world/plugin.yaml create mode 100644 plugins/wasm/hello-world/src/lib.rs diff --git a/plugins/wasm/.cargo/config.toml b/plugins/wasm/.cargo/config.toml new file mode 100644 index 00000000..9b923aa9 --- /dev/null +++ b/plugins/wasm/.cargo/config.toml @@ -0,0 +1,3 @@ +[build] +target = "wasm32-wasip1" + diff --git a/plugins/wasm/.gitignore b/plugins/wasm/.gitignore new file mode 100644 index 00000000..a788e5e5 --- /dev/null +++ b/plugins/wasm/.gitignore @@ -0,0 +1,4 @@ +/target/ +**/*.wasm +!README.md + diff --git a/plugins/wasm/Cargo.toml b/plugins/wasm/Cargo.toml new file mode 100644 index 00000000..4cbc3dfd --- /dev/null +++ b/plugins/wasm/Cargo.toml @@ -0,0 +1,21 @@ +[workspace] +members = [ + "hello-world", +] +resolver = "2" + +[workspace.package] +version = "0.0.0" +edition = "2021" +publish = false + +[workspace.dependencies] +proxy-wasm = "0.2" + +[profile.release] +codegen-units = 1 +opt-level = "z" +lto = "fat" +strip = true +panic = "abort" + diff --git a/plugins/wasm/README.md b/plugins/wasm/README.md new file mode 100644 index 00000000..ad24725e --- /dev/null +++ b/plugins/wasm/README.md @@ -0,0 +1,60 @@ +# Spacegate Wasm Plugins + +This directory is the dedicated development workspace for Spacegate Proxy-Wasm plugins. + +## Layout + +```text +plugins/wasm/ + Cargo.toml + hello-world/ + Cargo.toml + src/lib.rs + plugin.yaml +``` + +Use this directory for plugin source code. Keep compiled `.wasm` files in `resource/wasm/` for local demos, or publish them as OCI artifacts/images for Kubernetes usage. + +## Build + +Install the wasm target once: + +```bash +rustup target add wasm32-wasip1 +``` + +Build all plugins: + +```bash +cargo build --release --target wasm32-wasip1 --manifest-path plugins/wasm/Cargo.toml +``` + +The output for `hello-world` is: + +```text +plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_hello_world.wasm +``` + +If you run commands from inside `plugins/wasm/`, the local `.cargo/config.toml` already sets the wasm target: + +```bash +cd plugins/wasm +cargo build --release +``` + +For a local file-based demo, copy or package the built wasm into `resource/wasm/` and reference it with `file://...`. + +For production-style delivery, publish the wasm as an OCI artifact/image and reference it from a Higress-compatible `WasmPlugin`: + +```yaml +spec: + url: oci://registry.example.com/spacegate/plugins/hello-world:v1 +``` + +## Adding A Plugin + +1. Create `plugins/wasm//`. +2. Add it to `plugins/wasm/Cargo.toml` under `workspace.members`. +3. Set the crate type to `cdylib`. +4. Implement the Proxy-Wasm entry point with `proxy_wasm::main!`. +5. Add a `plugin.yaml` example that shows the intended `WasmPlugin` config. diff --git a/plugins/wasm/hello-world/Cargo.toml b/plugins/wasm/hello-world/Cargo.toml new file mode 100644 index 00000000..9489d84b --- /dev/null +++ b/plugins/wasm/hello-world/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "spacegate_plugin_hello_world" +version.workspace = true +edition.workspace = true +publish.workspace = true +description = "Hello World Proxy-Wasm plugin template for Spacegate." + +[lib] +crate-type = ["cdylib"] + +[dependencies] +proxy-wasm.workspace = true + diff --git a/plugins/wasm/hello-world/plugin.yaml b/plugins/wasm/hello-world/plugin.yaml new file mode 100644 index 00000000..68c87c7a --- /dev/null +++ b/plugins/wasm/hello-world/plugin.yaml @@ -0,0 +1,13 @@ +apiVersion: extensions.higress.io/v1alpha1 +kind: WasmPlugin +metadata: + name: hello-world + namespace: spacegate +spec: + url: oci://registry.example.com/spacegate/plugins/hello-world:v1 + pluginName: hello-world + phase: AUTHN + priority: 100 + failStrategy: FAIL_OPEN + defaultConfig: + message: hello world diff --git a/plugins/wasm/hello-world/src/lib.rs b/plugins/wasm/hello-world/src/lib.rs new file mode 100644 index 00000000..af93d62e --- /dev/null +++ b/plugins/wasm/hello-world/src/lib.rs @@ -0,0 +1,50 @@ +use proxy_wasm::hostcalls; +use proxy_wasm::traits::*; +use proxy_wasm::types::*; + +proxy_wasm::main! {{ + proxy_wasm::set_log_level(LogLevel::Info); + proxy_wasm::set_root_context(|_| -> Box { Box::new(HelloWorldRoot) }); +}} + +struct HelloWorldRoot; + +impl Context for HelloWorldRoot {} + +impl RootContext for HelloWorldRoot { + fn on_vm_start(&mut self, _: usize) -> bool { + let _ = hostcalls::log(LogLevel::Info, "hello world wasm plugin started"); + true + } + + fn on_configure(&mut self, _: usize) -> bool { + let _ = hostcalls::log(LogLevel::Info, "hello world wasm plugin configured"); + true + } + + fn create_http_context(&self, _: u32) -> Option> { + Some(Box::new(HelloWorldHttp)) + } + + fn get_type(&self) -> Option { + Some(ContextType::HttpContext) + } +} + +struct HelloWorldHttp; + +impl Context for HelloWorldHttp {} + +impl HttpContext for HelloWorldHttp { + fn on_http_request_headers(&mut self, _: usize, _: bool) -> Action { + let _ = hostcalls::log(LogLevel::Info, "hello world request reached wasm plugin"); + self.add_http_request_header("x-spacegate-wasm-plugin", "hello-world"); + + if self.get_http_request_header(":path").as_deref() == Some("/hello-world") { + self.send_http_response(200, vec![("content-type", "text/plain"), ("x-powered-by", "spacegate-wasm")], Some(b"hello world\n")); + return Action::Pause; + } + + Action::Continue + } +} From 3e2e55af6e6be5bca59cc2d44b17e59527a0b83d Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Wed, 20 May 2026 18:12:03 +0800 Subject: [PATCH 06/19] feat(wasm): add vm pool scheduling and resource isolation --- crates/plugin-wasm/Cargo.toml | 2 +- crates/plugin-wasm/src/config.rs | 50 +++++ crates/plugin-wasm/src/engine.rs | 30 ++- crates/plugin-wasm/src/error.rs | 10 + crates/plugin-wasm/src/host_fn.rs | 53 +++++- crates/plugin-wasm/src/host_state.rs | 29 ++- crates/plugin-wasm/src/lib.rs | 20 +- crates/plugin-wasm/src/shell.rs | 200 ++++++++++++++++---- crates/plugin-wasm/src/vm.rs | 71 ++++++- crates/plugin-wasm/tests/http_call.rs | 127 ++++++++++++- crates/plugin-wasm/tests/on_tick.rs | 6 +- crates/plugin-wasm/tests/spec_compliance.rs | 10 + 12 files changed, 545 insertions(+), 63 deletions(-) diff --git a/crates/plugin-wasm/Cargo.toml b/crates/plugin-wasm/Cargo.toml index f5d92ac3..b8587fd3 100644 --- a/crates/plugin-wasm/Cargo.toml +++ b/crates/plugin-wasm/Cargo.toml @@ -31,7 +31,7 @@ sha2 = "0.10" # host fn: dispatch_http_call 走 reqwest 异步客户端（用 0.12 与 spacegate 的 http=1 对齐） reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } # 异步驱动 + 待回填 dispatch 状态 -tokio = { workspace = true, features = ["sync", "macros", "rt"] } +tokio = { workspace = true, features = ["sync", "macros", "rt", "time"] } # inner.call 拿到的是 hyper Response，host fn 操作 header 时要用 http 类型 hyper = { workspace = true } http = "1" diff --git a/crates/plugin-wasm/src/config.rs b/crates/plugin-wasm/src/config.rs index a1ac0643..1bedec50 100644 --- a/crates/plugin-wasm/src/config.rs +++ b/crates/plugin-wasm/src/config.rs @@ -13,10 +13,21 @@ pub enum FailStrategy { #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct WasmLimits { + /// 单个 VM 的线性内存页数上限（1 page = 64KiB）。 #[serde(default)] pub max_memory_pages: Option, + /// 每次 guest hook 调用前补充的 fuel；默认不配置时使用近似无限预算。 #[serde(default)] pub fuel_per_call: Option, + /// 每次 guest hook 的 epoch 超时窗口，单位毫秒；依赖 host 的 1ms epoch ticker。 + #[serde(default)] + pub epoch_timeout_millis: Option, + /// host 需要物化 body 时允许的最大字节数，覆盖请求 body、响应 body、dispatch 请求/响应 body。 + #[serde(default)] + pub max_body_bytes: Option, + /// 单个 VM 同时允许的未完成 `proxy_http_call` 数量。 + #[serde(default)] + pub max_pending_calls: Option, } #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -42,6 +53,10 @@ fn default_use_cache() -> bool { true } +fn default_vm_pool_size() -> usize { + 1 +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] pub struct WasmPluginShellConfig { @@ -91,6 +106,18 @@ pub struct WasmPluginShellConfig { /// 暴露给 guest 的 `plugin_vm_id` well-known property；同时用于 `proxy_resolve_shared_queue`。 #[serde(default = "default_vm_id")] pub plugin_vm_id: String, + /// 同一个 wasm 插件实例内创建的 VM 数量。 + /// + /// 默认 1，保持单 VM 串行语义；设置为大于 1 后，多个独立 VM 共享同一个已编译 Module， + /// 请求按 try-lock + round-robin 分发，用于降低长时间 `dispatch_http_call` 对后续请求的阻塞。 + #[serde(default = "default_vm_pool_size")] + pub vm_pool_size: usize, + /// wait 策略专用 VM 池大小。 + /// + /// 默认 0，表示不启用分类调度，所有请求都进入普通 VM 池。设置为大于 0 后， + /// 带 `X-RateLimit-Policy: wait` 的请求会进入独立 wait 池，避免长等待请求占满普通池。 + #[serde(default)] + pub wait_vm_pool_size: usize, } fn default_vm_id() -> String { @@ -117,11 +144,34 @@ impl Default for WasmPluginShellConfig { plugin_name: String::new(), plugin_root_id: String::new(), plugin_vm_id: default_vm_id(), + vm_pool_size: default_vm_pool_size(), + wait_vm_pool_size: 0, } } } impl WasmPluginShellConfig { + pub fn normalized_vm_pool_size(&self) -> usize { + self.vm_pool_size.clamp(1, 64) + } + + pub fn normalized_wait_vm_pool_size(&self) -> usize { + self.wait_vm_pool_size.min(64) + } + + pub fn max_memory_bytes(&self) -> Option { + self.limits.max_memory_pages.map(|pages| pages as usize * 64 * 1024) + } + + pub fn guest_fuel_per_call(&self) -> u64 { + self.limits.fuel_per_call.unwrap_or(u64::MAX / 4).max(1) + } + + pub fn guest_epoch_deadline_ticks(&self) -> u64 { + // epoch ticker 以 1ms 为一跳；默认给一个很大的窗口，相当于不主动超时。 + self.limits.epoch_timeout_millis.unwrap_or(24 * 60 * 60 * 1000).clamp(1, 24 * 60 * 60 * 1000) + } + /// 把 `plugin_config`（任意 JSON）转换为 hai 风格 YAML 字节流。 /// /// hai-process-mix 在 `on_configure` 内是 `serde_yaml::from_slice::(&bytes)`， diff --git a/crates/plugin-wasm/src/engine.rs b/crates/plugin-wasm/src/engine.rs index e62d2060..0d5973cb 100644 --- a/crates/plugin-wasm/src/engine.rs +++ b/crates/plugin-wasm/src/engine.rs @@ -5,20 +5,44 @@ //! `proxy_http_call` 的异步语义通过 `tokio::spawn` + mpsc channel 实现， //! 不需要把整个 store 切到 async。 //! -//! 资源/超时限制（fuel/epoch）暂未启用：演进文档 §4.7 的"资源/Panic 隔离" -//! 列入后续阶段；本阶段优先保证 hai-process-mix 鉴权流程跑通。 - use once_cell::sync::OnceCell; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; use wasmtime::{Config, Engine}; static ENGINE: OnceCell = OnceCell::new(); +static EPOCH_TICKER_STARTED: AtomicBool = AtomicBool::new(false); /// 进程级单例 Engine（multi-memory 开，async 关）。 pub fn shared_engine() -> &'static Engine { ENGINE.get_or_init(|| { let mut cfg = Config::new(); cfg.wasm_multi_memory(true); + cfg.consume_fuel(true); + cfg.epoch_interruption(true); cfg.async_support(false); Engine::new(&cfg).expect("wasmtime Engine::new") }) } + +/// 启动一个进程级 epoch ticker。每 1ms 递增一次 Engine epoch，配合 Store epoch deadline +/// 给同步 guest hook 提供粗粒度墙钟超时保护。 +pub fn ensure_epoch_ticker_started() { + if EPOCH_TICKER_STARTED.load(Ordering::Acquire) { + return; + } + let Ok(handle) = tokio::runtime::Handle::try_current() else { + return; + }; + if EPOCH_TICKER_STARTED.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire).is_err() { + return; + } + let engine = shared_engine().clone(); + handle.spawn(async move { + let mut interval = tokio::time::interval(Duration::from_millis(1)); + loop { + interval.tick().await; + engine.increment_epoch(); + } + }); +} diff --git a/crates/plugin-wasm/src/error.rs b/crates/plugin-wasm/src/error.rs index bf584687..ae6303ea 100644 --- a/crates/plugin-wasm/src/error.rs +++ b/crates/plugin-wasm/src/error.rs @@ -16,6 +16,16 @@ pub enum WasmHostError { GuestTrap { hook: &'static str, source: wasmtime::Error }, #[error("dispatch_http_call: {0}")] Dispatch(String), + #[error("body too large: {actual} bytes exceeds limit {limit} bytes")] + BodyTooLarge { actual: usize, limit: usize }, + #[error("resource limit: {0}")] + ResourceLimit(String), #[error("config: {0}")] Config(String), } + +impl WasmHostError { + pub fn requires_vm_rebuild(&self) -> bool { + matches!(self, Self::GuestTrap { .. } | Self::Wasmtime(_) | Self::Dispatch(_) | Self::ResourceLimit(_)) + } +} diff --git a/crates/plugin-wasm/src/host_fn.rs b/crates/plugin-wasm/src/host_fn.rs index 0c682baf..79c494d5 100644 --- a/crates/plugin-wasm/src/host_fn.rs +++ b/crates/plugin-wasm/src/host_fn.rs @@ -604,6 +604,12 @@ fn register_http_call(linker: &mut Linker, dispatch_tx: tokio::sync:: }; let headers_bytes = mem.read_bytes(caller.as_context(), headers_data as u32, headers_size as u32).unwrap_or_default(); let body = if body_size > 0 { + if let Some(limit) = caller.data().shell_cfg.limits.max_body_bytes { + if body_size as usize > limit { + warn!(target: "spacegate_plugin_wasm", body_size, limit, "dispatch_http_call: request body exceeds max_body_bytes"); + return Status::BadArgument.as_i32(); + } + } mem.read_bytes(caller.as_context(), body_data as u32, body_size as u32).unwrap_or_default() } else { Vec::new() @@ -636,6 +642,17 @@ fn register_http_call(linker: &mut Linker, dispatch_tx: tokio::sync:: warn!(target: "spacegate_plugin_wasm", cluster = %cluster, "dispatch_http_call: cluster not configured"); return Status::BadArgument.as_i32(); }; + if let Some(limit) = caller.data().shell_cfg.limits.max_pending_calls { + if caller.data().pending_calls.len() >= limit { + warn!( + target: "spacegate_plugin_wasm", + pending_calls = caller.data().pending_calls.len(), + limit, + "dispatch_http_call: max_pending_calls reached" + ); + return Status::InternalFailure.as_i32(); + } + } let url = format!("{}{}", base.trim_end_matches('/'), path); let token = caller.data_mut().next_dispatch_token(); let source_ctx = caller.data().effective_context; @@ -647,6 +664,7 @@ fn register_http_call(linker: &mut Linker, dispatch_tx: tokio::sync:: }, ); let client = caller.data().http_client.clone(); + let max_body_bytes = caller.data().shell_cfg.limits.max_body_bytes; let timeout = Duration::from_millis(timeout_ms.max(1) as u64); let tx = dispatch_tx.clone(); tokio::spawn(async move { @@ -676,11 +694,36 @@ fn register_http_call(linker: &mut Linker, dispatch_tx: tokio::sync:: } } let body_bytes = resp.bytes().await.unwrap_or_default(); - HttpCallResult { - status, - status_message, - headers: hdrs, - body: body_bytes, + if let Some(limit) = max_body_bytes { + if body_bytes.len() > limit { + warn!( + target: "spacegate_plugin_wasm", + %url, + body_len = body_bytes.len(), + limit, + "dispatch_http_call response exceeds max_body_bytes" + ); + HttpCallResult { + status: 0, + status_message: format!("dispatch_http_call response body too large: {} > {limit}", body_bytes.len()), + headers: HeaderMap::new(), + body: Bytes::new(), + } + } else { + HttpCallResult { + status, + status_message, + headers: hdrs, + body: body_bytes, + } + } + } else { + HttpCallResult { + status, + status_message, + headers: hdrs, + body: body_bytes, + } } } Err(e) => { diff --git a/crates/plugin-wasm/src/host_state.rs b/crates/plugin-wasm/src/host_state.rs index 801acc72..63f315a6 100644 --- a/crates/plugin-wasm/src/host_state.rs +++ b/crates/plugin-wasm/src/host_state.rs @@ -13,7 +13,7 @@ use std::sync::Arc; use bytes::Bytes; use http::HeaderMap; -use wasmtime::{Memory, TypedFunc}; +use wasmtime::{Memory, ResourceLimiter, TypedFunc}; use crate::config::WasmPluginShellConfig; @@ -70,6 +70,29 @@ pub struct PendingCall { pub source_context_id: u32, } +#[derive(Debug, Clone)] +pub struct HostResourceLimiter { + max_memory_bytes: Option, +} + +impl HostResourceLimiter { + pub fn new(shell_cfg: &WasmPluginShellConfig) -> Self { + Self { + max_memory_bytes: shell_cfg.max_memory_bytes(), + } + } +} + +impl ResourceLimiter for HostResourceLimiter { + fn memory_growing(&mut self, _current: usize, desired: usize, _maximum: Option) -> wasmtime::Result { + Ok(self.max_memory_bytes.map(|max| desired <= max).unwrap_or(true)) + } + + fn table_growing(&mut self, _current: u32, _desired: u32, _maximum: Option) -> wasmtime::Result { + Ok(true) + } +} + /// 单个 HTTP 请求的所有状态（请求/响应头 / body / 上次 dispatch 结果 / 本地响应 / 短路标记）。 #[derive(Debug, Default)] pub struct RequestContext { @@ -143,6 +166,8 @@ pub struct HostState { pub plugin_name: String, pub plugin_root_id: String, pub plugin_vm_id: String, + /// Wasmtime 资源限制器：控制单 VM 线性内存增长。 + pub resource_limiter: HostResourceLimiter, } impl HostState { @@ -152,6 +177,7 @@ impl HostState { let plugin_name = shell_cfg.plugin_name.clone(); let plugin_root_id = shell_cfg.plugin_root_id.clone(); let plugin_vm_id = shell_cfg.plugin_vm_id.clone(); + let resource_limiter = HostResourceLimiter::new(&shell_cfg); Self { shell_cfg, configuration, @@ -170,6 +196,7 @@ impl HostState { plugin_name, plugin_root_id, plugin_vm_id, + resource_limiter, } } diff --git a/crates/plugin-wasm/src/lib.rs b/crates/plugin-wasm/src/lib.rs index e4db8e71..7fa11892 100644 --- a/crates/plugin-wasm/src/lib.rs +++ b/crates/plugin-wasm/src/lib.rs @@ -11,7 +11,7 @@ //! allocator 优先 `proxy_on_memory_allocate`，否则回退 `malloc`。 //! - **Logging**：`proxy_log` / `proxy_get_log_level` 完整实现（host tracing 级别映射）。 //! - **Clocks**：`proxy_get_current_time_nanoseconds` + `wasi_snapshot_preview1.clock_time_get`。 -//! - **Timers**：`proxy_set_tick_period_milliseconds` 完整生效；`shell.rs` 起一条 50ms 颗粒度的 +//! - **Timers**：`proxy_set_tick_period_milliseconds` 完整生效；`shell.rs` 为每个 Vm 起一条 50ms 颗粒度的 //! 后台 tokio 任务，到点 → `Vm::tick()` → guest `proxy_on_tick`。这要求 `Plugin::create` //! 时存在 tokio runtime（spacegate-shell 的标准启动路径）；无 runtime 时降级为不驱动。 //! - **Randomness**：`wasi_snapshot_preview1.random_get` 走 `getrandom`（OS RNG）。 @@ -37,18 +37,26 @@ //! - **gRPC**：按 spec 全部 `Unimplemented`。 //! - **Foreign function**：按 spec `NotFound`（无注册表）。 //! - **`proxy_done` / `proxy_set_effective_context`**：完整实现。 +//! - **资源隔离**：`limits.max_memory_pages` 通过 Wasmtime `ResourceLimiter` 限制线性内存增长； +//! `limits.fuel_per_call` 和 `limits.epoch_timeout_millis` 在每次 guest hook 前重置执行预算； +//! `limits.max_body_bytes` 限制 host 物化 request/response/dispatch body 的大小； +//! `limits.max_pending_calls` 限制单 VM 未完成 `proxy_http_call` 数。 //! //! ## Guest callbacks driven by host //! -//! - 启动：`_initialize`/`_start` → `proxy_on_context_create(root,0)` → -//! `proxy_on_vm_start` → `proxy_on_configure`（**仅一次**，由 `WasmPluginShell::create` 执行）。 +//! - 启动：每个 Vm 执行 `_initialize`/`_start` → `proxy_on_context_create(root,0)` → +//! `proxy_on_vm_start` → `proxy_on_configure`（由 `WasmPluginShell::create` 按 `vm_pool_size` 执行）。 //! - 每请求：`proxy_on_context_create(http_id, root)` → `proxy_on_request_headers` → //! （可选）`proxy_on_request_body` → （可选）`proxy_on_request_trailers` → //! `inner.call` → `proxy_on_response_headers` → （可选）`proxy_on_response_body` → //! （可选）`proxy_on_response_trailers` → `proxy_on_log` → `proxy_on_done` → `proxy_on_delete`。 -//! `WasmPluginShell` 持有 `Arc>`，所有请求串行经过同一 root VM -//! ——与 envoy/istio 的 per-worker 单线 wasm 模型一致。 -//! - 后台 `proxy_on_tick`：`shell.rs` 起 50ms 颗粒度的 tokio 任务驱动；guest 通过 +//! `WasmPluginShell` 默认持有 1 个 `Arc>`；配置 `vm_pool_size > 1` +//! 后会创建多个独立 root Vm，并通过 try-lock + round-robin 调度请求。 +//! 配置 `wait_vm_pool_size > 0` 后，带 `X-RateLimit-Policy: wait` 的请求会进入单独 wait VM 池， +//! 其余 `abandon`/`queue`/未标记请求仍进入普通 VM 池，避免长等待请求拖住普通限流路径。 +//! 每个 VM slot 会记录并输出 inflight tracing 字段；guest trap / 资源隔离错误 / dispatch 通道异常后， +//! shell 会在原 slot 内尝试重建 VM，避免异常 Store 长期留在池内。 +//! - 后台 `proxy_on_tick`：`shell.rs` 为每个 Vm 起 50ms 颗粒度的 tokio 任务驱动；guest 通过 //! `proxy_set_tick_period_milliseconds` 改周期。 //! - 异步 `proxy_on_http_call_response`：在 Pause 状态机里 await `dispatch_rx` 后回调。 //! - Pause/Continue：`proxy_continue_stream` 同步解除 Pause；多次 dispatch 可串联。 diff --git a/crates/plugin-wasm/src/shell.rs b/crates/plugin-wasm/src/shell.rs index 43428dec..228e6038 100644 --- a/crates/plugin-wasm/src/shell.rs +++ b/crates/plugin-wasm/src/shell.rs @@ -1,20 +1,23 @@ -//! `Plugin` 实现：实例化一次长生命 Vm，后续请求复用，并起一条后台 tick 任务驱动 `proxy_on_tick`。 +//! `Plugin` 实现：实例化一个或多个长生命 Vm，后续请求复用，并为每个 Vm 起一条后台 tick 任务驱动 `proxy_on_tick`。 //! -//! 与旧版「每请求新建 Vm」相比的取舍： +//! 与「每请求新建 Vm」相比的取舍： //! //! - 优点：guest 的 root context 可保留状态；`proxy_on_tick` 可真正按 `proxy_set_tick_period_milliseconds` 周期触发； //! `on_vm_start` / `on_configure` 仅跑一次，热路径少几毫秒。 -//! - 代价：所有经过本插件实例的请求会通过同一把 `tokio::sync::Mutex` 串行化处理—— -//! wasmtime `Store` 是 !Sync，无法并发；envoy / istio 的 proxy-wasm 实现也是相同模型。 -//! -//! 后续要做更细粒度并发（多 root VM 池）属于演进文档 §4.3 范畴，本版不在范围内。 +//! - 单个 `Vm` 内仍通过 `tokio::sync::Mutex` 串行化处理（wasmtime `Store` 是 !Sync）； +//! 配置 `vm_pool_size > 1` 时，通过多个独立 `Store + Instance` 提供插件实例内并发。 +//! - 配置 `wait_vm_pool_size > 0` 时，`X-RateLimit-Policy: wait` 请求会进入独立 wait 池； +//! 其他请求继续走普通池。 -use std::sync::Arc; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; use std::time::{Duration, Instant}; use spacegate_kernel::{SgBody, SgRequest, SgResponse}; use spacegate_plugin::{BoxError, Inner, Plugin, PluginConfig}; -use tokio::sync::Mutex as AsyncMutex; +use tokio::sync::{Mutex as AsyncMutex, MutexGuard}; use crate::config::{FailStrategy, WasmPluginShellConfig}; use crate::runtime::default_module_cache; @@ -28,15 +31,58 @@ impl Drop for AbortOnDrop { } } +#[derive(Clone)] +struct VmSlot { + vm: Arc>, + inflight: Arc, +} + +impl VmSlot { + fn new(vm: Vm) -> Self { + Self { + vm: Arc::new(AsyncMutex::new(vm)), + inflight: Arc::new(AtomicUsize::new(0)), + } + } +} + +struct InflightGuard { + inflight: Arc, + pool_name: &'static str, + vm_index: usize, +} + +impl InflightGuard { + fn new(slot: &VmSlot, pool_name: &'static str, vm_index: usize) -> Self { + let current = slot.inflight.fetch_add(1, Ordering::AcqRel) + 1; + tracing::debug!(target: "spacegate_plugin_wasm", vm_pool = pool_name, vm_index, inflight = current, "VM inflight incremented"); + Self { + inflight: slot.inflight.clone(), + pool_name, + vm_index, + } + } +} + +impl Drop for InflightGuard { + fn drop(&mut self) { + let current = self.inflight.fetch_sub(1, Ordering::AcqRel).saturating_sub(1); + tracing::debug!(target: "spacegate_plugin_wasm", vm_pool = self.pool_name, vm_index = self.vm_index, inflight = current, "VM inflight decremented"); + } +} + /// Proxy-Wasm 宿主壳插件（`CODE = "wasm"`）。 pub struct WasmPluginShell { cfg: Arc, #[allow(dead_code)] module: Arc, - vm: Arc>, + vms: Vec, + wait_vms: Vec, + next_vm: AtomicUsize, + next_wait_vm: AtomicUsize, /// 后台 tick 任务句柄；shell drop 时自动 abort。 /// `None` 表示创建时没有 tokio runtime 上下文（非测试常见路径），tick 退化为不驱动。 - _tick_task: Option, + _tick_tasks: Vec, } impl Plugin for WasmPluginShell { @@ -44,32 +90,42 @@ impl Plugin for WasmPluginShell { fn call(&self, req: SgRequest, inner: Inner) -> impl std::future::Future> + Send { let cfg = self.cfg.clone(); - let vm = self.vm.clone(); + let use_wait_pool = is_wait_policy(&req) && !self.wait_vms.is_empty(); + let pool_name = if use_wait_pool { "wait" } else { "normal" }; + let slots = if use_wait_pool { self.wait_vms.clone() } else { self.vms.clone() }; + let module = self.module.clone(); + let start_index = if use_wait_pool { + self.next_wait_vm.fetch_add(1, Ordering::Relaxed) + } else { + self.next_vm.fetch_add(1, Ordering::Relaxed) + }; async move { tracing::info!( target: "spacegate_plugin_wasm", method = %req.method(), uri = %req.uri(), + vm_pool = pool_name, "wasm plugin shell: request entered plugin layer" ); - let mut guard = vm.lock().await; - match guard.process(req, inner).await { - Ok(resp) => { - tracing::info!(target: "spacegate_plugin_wasm", status = %resp.status(), "Vm::process ok"); - Ok(resp) - } - Err(e) => { - tracing::error!(target: "spacegate_plugin_wasm", error = %e, "wasm plugin failed"); - let status = if matches!(cfg.fail_strategy, FailStrategy::FailOpen) { - http::StatusCode::BAD_GATEWAY - } else { - http::StatusCode::INTERNAL_SERVER_ERROR - }; - let mut resp = SgResponse::new(SgBody::full(format!("wasm plugin error: {e}"))); - *resp.status_mut() = status; - Ok(resp) + + if slots.is_empty() { + let mut resp = SgResponse::new(SgBody::full(format!("wasm plugin error: empty {pool_name} VM pool"))); + *resp.status_mut() = http::StatusCode::INTERNAL_SERVER_ERROR; + return Ok(resp); + } + + for offset in 0..slots.len() { + let index = start_index.wrapping_add(offset) % slots.len(); + if let Ok(guard) = slots[index].vm.try_lock() { + let _inflight = InflightGuard::new(&slots[index], pool_name, index); + return process_with_vm(module, cfg, req, inner, guard, pool_name, index).await; } } + + let index = start_index % slots.len(); + let _inflight = InflightGuard::new(&slots[index], pool_name, index); + let guard = slots[index].vm.lock().await; + process_with_vm(module, cfg, req, inner, guard, pool_name, index).await } } @@ -91,25 +147,101 @@ impl Plugin for WasmPluginShell { let cache = default_module_cache(); let module = cache.get_or_compile(&cfg).map_err(|e| -> BoxError { format!("compile wasm: {e}").into() })?; let cfg = Arc::new(cfg); - let vm = Vm::new(&module, cfg.clone()).map_err(|e| -> BoxError { format!("Vm::new: {e}").into() })?; - let vm = Arc::new(AsyncMutex::new(vm)); - let tick_task = spawn_tick_loop(&vm); + let pool_size = cfg.normalized_vm_pool_size(); + let wait_pool_size = cfg.normalized_wait_vm_pool_size(); + let mut vms = Vec::with_capacity(pool_size); + let mut wait_vms = Vec::with_capacity(wait_pool_size); + let mut tick_tasks = Vec::with_capacity(pool_size + wait_pool_size); + for index in 0..pool_size { + let vm = Vm::new(&module, cfg.clone()).map_err(|e| -> BoxError { format!("Vm::new[{index}]: {e}").into() })?; + let slot = VmSlot::new(vm); + if let Some(task) = spawn_tick_loop("normal", index, &slot.vm) { + tick_tasks.push(task); + } + vms.push(slot); + } + for index in 0..wait_pool_size { + let vm = Vm::new(&module, cfg.clone()).map_err(|e| -> BoxError { format!("Vm::new[wait:{index}]: {e}").into() })?; + let slot = VmSlot::new(vm); + if let Some(task) = spawn_tick_loop("wait", index, &slot.vm) { + tick_tasks.push(task); + } + wait_vms.push(slot); + } + tracing::info!( + target: "spacegate_plugin_wasm", + pool_size, + wait_pool_size, + "wasm plugin: VM pools created" + ); Ok(Self { cfg, module, - vm, - _tick_task: tick_task, + vms, + wait_vms, + next_vm: AtomicUsize::new(0), + next_wait_vm: AtomicUsize::new(0), + _tick_tasks: tick_tasks, }) } } +fn is_wait_policy(req: &SgRequest) -> bool { + req.headers().get("x-ratelimit-policy").and_then(|value| value.to_str().ok()).map(|value| value.trim().eq_ignore_ascii_case("wait")).unwrap_or(false) +} + +async fn process_with_vm( + module: Arc, + cfg: Arc, + req: SgRequest, + inner: Inner, + mut guard: MutexGuard<'_, Vm>, + pool_name: &'static str, + vm_index: usize, +) -> Result { + match guard.process(req, inner).await { + Ok(resp) => { + tracing::info!(target: "spacegate_plugin_wasm", vm_pool = pool_name, vm_index, status = %resp.status(), "Vm::process ok"); + Ok(resp) + } + Err(e) => { + tracing::error!(target: "spacegate_plugin_wasm", vm_pool = pool_name, vm_index, error = %e, "wasm plugin failed"); + if e.requires_vm_rebuild() { + match Vm::new(&module, cfg.clone()) { + Ok(new_vm) => { + *guard = new_vm; + tracing::warn!(target: "spacegate_plugin_wasm", vm_pool = pool_name, vm_index, "VM rebuilt after abnormal failure"); + } + Err(rebuild_err) => { + tracing::error!( + target: "spacegate_plugin_wasm", + vm_pool = pool_name, + vm_index, + error = %rebuild_err, + "VM rebuild failed after abnormal failure" + ); + } + } + } + let status = if matches!(cfg.fail_strategy, FailStrategy::FailOpen) { + http::StatusCode::BAD_GATEWAY + } else { + http::StatusCode::INTERNAL_SERVER_ERROR + }; + let mut resp = SgResponse::new(SgBody::full(format!("wasm plugin error: {e}"))); + *resp.status_mut() = status; + Ok(resp) + } + } +} + /// 起一条 50ms 粒度的轮询任务：每个 tick 看一眼 `Vm::tick_period_ms()`，到点了就 `Vm::tick()`。 /// /// - 粒度 50ms 是工程取舍：spec 没有规定 tick 必须精确，envoy 也是大颗粒度；如果 guest 设置 < 50ms 的周期， /// 实际触发率会被压到 50ms 一次——记入 `lib.rs` 顶部已知限制。 /// - 任务持有 `Arc>`，shell drop 时 `AbortOnDrop` 立刻 abort，不存在悬挂任务。 /// - 若 `proxy_on_tick` trap，记 error 后退出循环（防止热循环 panic）。 -fn spawn_tick_loop(vm: &Arc>) -> Option { +fn spawn_tick_loop(pool_name: &'static str, vm_index: usize, vm: &Arc>) -> Option { let handle = tokio::runtime::Handle::try_current().ok()?; let vm = vm.clone(); let task = handle.spawn(async move { @@ -136,6 +268,8 @@ fn spawn_tick_loop(vm: &Arc>) -> Option { if let Err(e) = guard.tick() { tracing::error!( target: "spacegate_plugin_wasm", + vm_pool = pool_name, + vm_index, error = %e, "proxy_on_tick failed; stopping tick task" ); diff --git a/crates/plugin-wasm/src/vm.rs b/crates/plugin-wasm/src/vm.rs index 84bbacbd..f9db0875 100644 --- a/crates/plugin-wasm/src/vm.rs +++ b/crates/plugin-wasm/src/vm.rs @@ -24,14 +24,14 @@ use std::sync::Arc; use bytes::Bytes; use http::HeaderMap; -use http_body_util::BodyExt; +use http_body_util::{BodyExt, Limited}; use spacegate_kernel::{SgBody, SgRequest, SgResponse}; use tracing::{debug, info, warn}; use wasmtime::{AsContext, AsContextMut, Instance, Linker, Store, TypedFunc}; use crate::abi::{Action, MemoryHelper}; use crate::config::{FailStrategy, WasmPluginShellConfig}; -use crate::engine::shared_engine; +use crate::engine::{ensure_epoch_ticker_started, shared_engine}; use crate::error::WasmHostError; use crate::host_fn::register_all; use crate::host_state::{ContextStage, HostState, HttpCallResult, PseudoHeaders, RequestContext}; @@ -70,9 +70,12 @@ impl Vm { /// `on_vm_start` / `on_configure` 全部是同步调用），所以 `WasmPluginShell::create` /// 这种 sync 上下文也能直接构造。 pub fn new(module: &wasmtime::Module, shell_cfg: Arc) -> Result { + ensure_epoch_ticker_started(); let engine = shared_engine(); let host = HostState::new(shell_cfg.clone()); let mut store: Store = Store::new(engine, host); + store.limiter(|state| &mut state.resource_limiter); + prepare_store_for_guest_call(&mut store)?; let mut linker: Linker = Linker::new(engine); let (dispatch_tx, dispatch_rx) = tokio::sync::mpsc::unbounded_channel::<(u32, HttpCallResult)>(); register_all(&mut linker, dispatch_tx).map_err(|e| WasmHostError::Instantiate(format!("register host fn: {e}")))?; @@ -82,6 +85,14 @@ impl Vm { let instance = linker.instantiate(&mut store, module).map_err(|e| WasmHostError::Instantiate(format!("instantiate: {e}")))?; let memory = instance.get_memory(&mut store, "memory").ok_or_else(|| WasmHostError::AbiViolation("no `memory` export".into()))?; + if let Some(max_pages) = shell_cfg.limits.max_memory_pages { + let current_pages = memory.size(&store) as u32; + if current_pages > max_pages { + return Err(WasmHostError::ResourceLimit(format!( + "initial memory pages {current_pages} exceeds max_memory_pages {max_pages}" + ))); + } + } store.data_mut().memory = Some(memory); // spec §Memory management：优先 `proxy_on_memory_allocate`，否则回退 `malloc`。 if let Ok(alloc) = instance.get_typed_func::(&mut store, "proxy_on_memory_allocate") { @@ -94,8 +105,10 @@ impl Vm { // spec §Integration：先 `_initialize`；若不存在尝试 `_start`。 if let Ok(init) = instance.get_typed_func::<(), ()>(&mut store, "_initialize") { + prepare_store_for_guest_call(&mut store)?; init.call(&mut store, ()).map_err(|e| WasmHostError::Instantiate(format!("_initialize: {e}")))?; } else if let Ok(start) = instance.get_typed_func::<(), ()>(&mut store, "_start") { + prepare_store_for_guest_call(&mut store)?; start.call(&mut store, ()).map_err(|e| WasmHostError::Instantiate(format!("_start: {e}")))?; } @@ -156,6 +169,7 @@ impl Vm { if let Some(ref f) = vm.fn_on_vm_start { vm.store.data_mut().effective_context = root_id; let cfg_len = vm.store.data().configuration.len() as u32; + prepare_store_for_guest_call(&mut vm.store)?; let ok = f.call(&mut vm.store, (root_id, cfg_len)).map_err(|e| WasmHostError::GuestTrap { hook: "on_vm_start", source: e })?; if ok == 0 { return Err(WasmHostError::Instantiate("guest on_vm_start returned 0 (=invalid VM configuration)".into())); @@ -165,6 +179,7 @@ impl Vm { let cfg_len = vm.store.data().configuration.len() as u32; tracing::info!(target: "spacegate_plugin_wasm", cfg_len, "calling proxy_on_configure"); let configure_fn = vm.fn_on_configure.clone(); + prepare_store_for_guest_call(&mut vm.store)?; let ok = configure_fn.call(&mut vm.store, (root_id, cfg_len)).map_err(|e| WasmHostError::GuestTrap { hook: "on_configure", source: e })?; if ok == 0 { warn!(target: "spacegate_plugin_wasm", "guest on_configure returned 0 (=invalid config)"); @@ -175,6 +190,7 @@ impl Vm { fn create_context(&mut self, ctx_id: u32, parent_id: u32) -> Result<(), WasmHostError> { self.store.data_mut().effective_context = ctx_id; let f = self.fn_on_context_create.clone(); + self.prepare_guest_call()?; f.call(&mut self.store, (ctx_id, parent_id)).map_err(|e| WasmHostError::GuestTrap { hook: "on_context_create", source: e, @@ -229,6 +245,7 @@ impl Vm { let num_headers = (self.store.data().contexts[&http_ctx_id].request_headers.len() + 4) as u32; let end_of_stream_for_headers: u32 = if want_request_body { 0 } else { 1 }; let on_req_hdr = self.fn_on_request_headers.clone(); + self.prepare_guest_call()?; let action_raw = on_req_hdr.call(&mut self.store, (http_ctx_id, num_headers, end_of_stream_for_headers)).map_err(|e| WasmHostError::GuestTrap { hook: "on_request_headers", source: e, @@ -249,10 +266,7 @@ impl Vm { // ─── on_request_body：把请求 body 物化后喂给 guest（仅当 guest 导出该 hook）─── let (new_req_for_inner, collected_body_after_hook) = if want_request_body { // collect body - let collected = match body.collect().await { - Ok(c) => c.to_bytes(), - Err(_) => Bytes::new(), - }; + let collected = collect_body_limited(body, self.store.data().shell_cfg.limits.max_body_bytes).await?; let body_size = collected.len() as u32; { let st = self.store.data_mut(); @@ -265,6 +279,7 @@ impl Vm { } } let on_req_body = self.fn_on_request_body.clone().expect("guarded by want_request_body"); + self.prepare_guest_call()?; let action_raw = on_req_body.call(&mut self.store, (http_ctx_id, body_size, 1)).map_err(|e| WasmHostError::GuestTrap { hook: "on_request_body", source: e, @@ -290,6 +305,7 @@ impl Vm { ctx.stage = ContextStage::RequestTrailers; ctx.continue_requested = false; } + self.prepare_guest_call()?; let action_raw = f.call(&mut self.store, (http_ctx_id, 0)).map_err(|e| WasmHostError::GuestTrap { hook: "on_request_trailers", source: e, @@ -346,6 +362,7 @@ impl Vm { let want_response_body = self.fn_on_response_body.is_some(); let end_of_stream_for_resp_hdr: u32 = if want_response_body { 0 } else { 1 }; let on_resp_hdr = self.fn_on_response_headers.clone(); + self.prepare_guest_call()?; let action_raw = on_resp_hdr.call(&mut self.store, (http_ctx_id, (resp_headers.len() + 1) as u32, end_of_stream_for_resp_hdr)).map_err(|e| WasmHostError::GuestTrap { hook: "on_response_headers", source: e, @@ -361,10 +378,7 @@ impl Vm { // ─── on_response_body ─── let (mut final_headers, final_body): (HeaderMap, SgBody) = if let Some(f) = self.fn_on_response_body.clone() { - let collected = match resp_body.collect().await { - Ok(c) => c.to_bytes(), - Err(_) => Bytes::new(), - }; + let collected = collect_body_limited(resp_body, self.store.data().shell_cfg.limits.max_body_bytes).await?; let body_size = collected.len() as u32; { let st = self.store.data_mut(); @@ -376,6 +390,7 @@ impl Vm { st.effective_context = http_ctx_id; } } + self.prepare_guest_call()?; let action_raw = f.call(&mut self.store, (http_ctx_id, body_size, 1)).map_err(|e| WasmHostError::GuestTrap { hook: "on_response_body", source: e, @@ -397,6 +412,7 @@ impl Vm { ctx.stage = ContextStage::ResponseTrailers; ctx.continue_requested = false; } + self.prepare_guest_call()?; let _ = f.call(&mut self.store, (http_ctx_id, 0)).map_err(|e| WasmHostError::GuestTrap { hook: "on_response_trailers", source: e, @@ -452,6 +468,7 @@ impl Vm { } debug!(target: "spacegate_plugin_wasm", token, source_ctx_id, status = result.status, body_len, "fire proxy_on_http_call_response"); let f = self.fn_on_http_call_response.clone(); + self.prepare_guest_call()?; f.call(&mut self.store, (source_ctx_id, token, header_count, body_len, 0)).map_err(|e| WasmHostError::GuestTrap { hook: "on_http_call_response", source: e, @@ -465,6 +482,7 @@ impl Vm { ctx.stage = ContextStage::Log; } if let Some(f) = self.fn_on_log.clone() { + self.prepare_guest_call()?; let _ = f.call(&mut self.store, ctx_id); } if let Some(f) = self.fn_on_done.clone() { @@ -475,6 +493,7 @@ impl Vm { if let Some(ctx) = self.store.data_mut().contexts.get_mut(&ctx_id) { ctx.awaiting_done = true; } + self.prepare_guest_call()?; let v = f.call(&mut self.store, ctx_id).unwrap_or(1); let done = v != 0 || self.store.data().contexts.get(&ctx_id).map(|c| c.done_marker).unwrap_or(true); if !done { @@ -486,6 +505,7 @@ impl Vm { } } if let Some(f) = self.fn_on_delete.clone() { + self.prepare_guest_call()?; let _ = f.call(&mut self.store, ctx_id); } self.store.data_mut().contexts.remove(&ctx_id); @@ -509,9 +529,40 @@ impl Vm { return Ok(()); }; self.store.data_mut().effective_context = self.root_id; + self.prepare_guest_call()?; f.call(&mut self.store, self.root_id).map_err(|e| WasmHostError::GuestTrap { hook: "on_tick", source: e })?; Ok(()) } + + fn prepare_guest_call(&mut self) -> Result<(), WasmHostError> { + prepare_store_for_guest_call(&mut self.store) + } +} + +fn prepare_store_for_guest_call(store: &mut Store) -> Result<(), WasmHostError> { + let fuel = store.data().shell_cfg.guest_fuel_per_call(); + store.set_fuel(fuel).map_err(|e| WasmHostError::ResourceLimit(format!("set fuel: {e}")))?; + let deadline = store.data().shell_cfg.guest_epoch_deadline_ticks(); + store.set_epoch_deadline(deadline); + store.epoch_deadline_trap(); + Ok(()) +} + +async fn collect_body_limited(body: SgBody, limit: Option) -> Result { + if let Some(limit) = limit { + let limited = Limited::new(body, limit); + let collected = limited.collect().await.map_err(|_| WasmHostError::BodyTooLarge { + actual: limit.saturating_add(1), + limit, + })?; + let bytes = collected.to_bytes(); + if bytes.len() > limit { + return Err(WasmHostError::BodyTooLarge { actual: bytes.len(), limit }); + } + Ok(bytes) + } else { + Ok(body.collect().await.map(|c| c.to_bytes()).unwrap_or_default()) + } } fn rebuild_uri(scheme: &str, authority: &str, path: &str) -> Option { diff --git a/crates/plugin-wasm/tests/http_call.rs b/crates/plugin-wasm/tests/http_call.rs index d54fef12..b7022764 100644 --- a/crates/plugin-wasm/tests/http_call.rs +++ b/crates/plugin-wasm/tests/http_call.rs @@ -12,7 +12,7 @@ use std::convert::Infallible; use std::net::SocketAddr; use std::path::PathBuf; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; use bytes::Bytes; use http_body_util::{BodyExt, Full}; @@ -23,9 +23,11 @@ use hyper_util::rt::TokioIo; use spacegate_kernel::backend_service::ArcHyperService; use spacegate_kernel::helper_layers::function::Inner; use spacegate_kernel::{SgBody, SgRequest, SgResponse}; +use spacegate_plugin::{Plugin, PluginConfig, PluginInstanceId, PluginInstanceName}; use spacegate_plugin_wasm::config::WasmPluginShellConfig; use spacegate_plugin_wasm::engine::shared_engine; use spacegate_plugin_wasm::vm::Vm; +use spacegate_plugin_wasm::WasmPluginShell; use tokio::net::TcpListener; use wasmtime::Module; @@ -100,6 +102,29 @@ async fn start_mock_server(body_byte: u8) -> SocketAddr { addr } +async fn start_delayed_mock_server(body_byte: u8, delay: Duration) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("local_addr"); + tokio::spawn(async move { + loop { + let (stream, _) = match listener.accept().await { + Ok(s) => s, + Err(_) => return, + }; + tokio::spawn(async move { + let svc = service_fn(move |_req: HyperRequest| async move { + tokio::time::sleep(delay).await; + let body = Bytes::from(vec![body_byte]); + let resp = Response::builder().status(200).body(Full::new(body)).expect("build resp"); + Ok::<_, Infallible>(resp) + }); + let _ = http1::Builder::new().serve_connection(TokioIo::new(stream), svc).await; + }); + } + }); + addr +} + // ───────────────────────────────────────────────────────── // mock inner.call：guest 放行后会下沉到这里，echo body 即可 // ───────────────────────────────────────────────────────── @@ -151,6 +176,18 @@ async fn run(auth_byte: u8) -> (u16, Bytes) { (resp.status().as_u16(), body) } +fn protected_request() -> SgRequest { + protected_request_with_policy(None) +} + +fn protected_request_with_policy(policy: Option<&str>) -> SgRequest { + let mut builder = HyperRequest::builder().method("POST").uri("http://example.test/").header("host", "example.test"); + if let Some(policy) = policy { + builder = builder.header("x-ratelimit-policy", policy); + } + builder.body(SgBody::full(Bytes::from_static(b"protected payload"))).expect("build req") +} + // ───────────────────────────────────────────────────────── // auth byte < threshold → 放行；echo 回原 body // ───────────────────────────────────────────────────────── @@ -172,3 +209,91 @@ async fn auth_random_deny() { assert_eq!(status, 403, "expected deny → 403"); assert_eq!(body, Bytes::from_static(b"forbidden")); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn vm_pool_runs_slow_http_calls_concurrently() { + let wasm = ensure_guest_built(); + let addr = start_delayed_mock_server(50, Duration::from_millis(450)).await; + tokio::time::sleep(Duration::from_millis(20)).await; + + let shell = WasmPluginShell::create(PluginConfig { + id: PluginInstanceId { + code: "wasm".into(), + name: PluginInstanceName::named("vm-pool-test"), + }, + spec: serde_json::json!({ + "url": format!("file://{}", wasm.display()), + "plugin_config": { + "mode": "auth_random", + "auth_cluster": "auth", + "auth_threshold": 128 + }, + "clusters": { + "auth": format!("http://{addr}") + }, + "vm_pool_size": 2 + }), + }) + .expect("create wasm shell"); + + let started = Instant::now(); + let (resp1, resp2) = tokio::join!(shell.call(protected_request(), echo_inner()), shell.call(protected_request(), echo_inner())); + let elapsed = started.elapsed(); + + let (resp1, body1) = full_body(resp1.expect("resp1")).await; + let (resp2, body2) = full_body(resp2.expect("resp2")).await; + assert_eq!(resp1.status(), http::StatusCode::OK); + assert_eq!(resp2.status(), http::StatusCode::OK); + assert_eq!(body1, Bytes::from_static(b"protected payload")); + assert_eq!(body2, Bytes::from_static(b"protected payload")); + assert!( + elapsed < Duration::from_millis(800), + "expected two 450ms dispatches to overlap with vm_pool_size=2, elapsed={elapsed:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn wait_policy_uses_separate_vm_pool() { + let wasm = ensure_guest_built(); + let addr = start_delayed_mock_server(50, Duration::from_millis(450)).await; + tokio::time::sleep(Duration::from_millis(20)).await; + + let shell = WasmPluginShell::create(PluginConfig { + id: PluginInstanceId { + code: "wasm".into(), + name: PluginInstanceName::named("wait-vm-pool-test"), + }, + spec: serde_json::json!({ + "url": format!("file://{}", wasm.display()), + "plugin_config": { + "mode": "auth_random", + "auth_cluster": "auth", + "auth_threshold": 128 + }, + "clusters": { + "auth": format!("http://{addr}") + }, + "vm_pool_size": 1, + "wait_vm_pool_size": 1 + }), + }) + .expect("create wasm shell"); + + let started = Instant::now(); + let (wait_resp, normal_resp) = tokio::join!(shell.call(protected_request_with_policy(Some("wait")), echo_inner()), async { + tokio::time::sleep(Duration::from_millis(50)).await; + shell.call(protected_request(), echo_inner()).await + }); + let elapsed = started.elapsed(); + + let (wait_resp, wait_body) = full_body(wait_resp.expect("wait resp")).await; + let (normal_resp, normal_body) = full_body(normal_resp.expect("normal resp")).await; + assert_eq!(wait_resp.status(), http::StatusCode::OK); + assert_eq!(normal_resp.status(), http::StatusCode::OK); + assert_eq!(wait_body, Bytes::from_static(b"protected payload")); + assert_eq!(normal_body, Bytes::from_static(b"protected payload")); + assert!( + elapsed < Duration::from_millis(800), + "expected wait traffic to use wait_vm_pool and not block normal pool, elapsed={elapsed:?}" + ); +} diff --git a/crates/plugin-wasm/tests/on_tick.rs b/crates/plugin-wasm/tests/on_tick.rs index a3533bd2..0cc316ec 100644 --- a/crates/plugin-wasm/tests/on_tick.rs +++ b/crates/plugin-wasm/tests/on_tick.rs @@ -84,11 +84,11 @@ async fn proxy_on_tick_drives_background_ticks() { // shell 内部已经 spawn 了 50ms 颗粒度的 tick 任务； // 期间 guest `on_vm_start` 把 period 设成 50ms。 - // 等 300ms 至少 4 次 tick（保留调度抖动余量）。 - tokio::time::sleep(Duration::from_millis(300)).await; + // 等 450ms 至少 4 次 tick（保留 CI / 本地调度抖动余量）。 + tokio::time::sleep(Duration::from_millis(450)).await; let count = read_counter(); - assert!(count >= 4, "expected >= 4 ticks in 300ms, got {count}"); + assert!(count >= 4, "expected >= 4 ticks in 450ms, got {count}"); tracing::info!("got {count} ticks"); // 取一次 snapshot，drop 之后再 sleep 同等时间，断言不再继续增长（允许 1 次余量： diff --git a/crates/plugin-wasm/tests/spec_compliance.rs b/crates/plugin-wasm/tests/spec_compliance.rs index c9491de8..90273454 100644 --- a/crates/plugin-wasm/tests/spec_compliance.rs +++ b/crates/plugin-wasm/tests/spec_compliance.rs @@ -109,6 +109,7 @@ impl GuestVm { host.effective_context = HTTP_CONTEXT_ID; let mut store: Store = Store::new(engine, host); + prepare_guest_budget(&mut store); let mut linker: Linker = Linker::new(engine); // dispatch_tx 在本测试里不会被消费——保留 rx 不让通道关闭即可。 let (dispatch_tx, _dispatch_rx) = tokio::sync::mpsc::unbounded_channel::<(u32, HttpCallResult)>(); @@ -128,8 +129,10 @@ impl GuestVm { // _initialize 优先（SDK 在 wasm32-wasip1 上默认导这个），回退 _start。 if let Ok(init) = instance.get_typed_func::<(), ()>(&mut store, "_initialize") { + prepare_guest_budget(&mut store); init.call(&mut store, ()).expect("_initialize"); } else if let Ok(start) = instance.get_typed_func::<(), ()>(&mut store, "_start") { + prepare_guest_budget(&mut store); start.call(&mut store, ()).expect("_start"); } @@ -138,6 +141,7 @@ impl GuestVm { fn run_test(&mut self, scenario: u32) -> u32 { let f: TypedFunc = self.instance.get_typed_func(&mut self.store, "__run_test").expect("__run_test export"); + prepare_guest_budget(&mut self.store); f.call(&mut self.store, scenario).expect("__run_test trap-free") } @@ -146,6 +150,12 @@ impl GuestVm { } } +fn prepare_guest_budget(store: &mut Store) { + store.set_fuel(u64::MAX / 4).expect("set test fuel"); + store.set_epoch_deadline(24 * 60 * 60 * 1000); + store.epoch_deadline_trap(); +} + // ───────────────────────────────────────────────────────── // 唯一一个 `#[test]` —— 跑完所有 scenario；隔离 shared/queue/metric 已通过 scenario 内独立 key 实现。 // ───────────────────────────────────────────────────────── From 67fa7cfad6579ee750591e32d66da77eecc60512 Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Wed, 20 May 2026 19:50:32 +0800 Subject: [PATCH 07/19] feat: add ai gateway queue plugin --- Cargo.toml | 1 + binary/ai-gateway-service/Cargo.toml | 29 + binary/ai-gateway-service/README.md | 41 ++ binary/ai-gateway-service/src/main.rs | 581 ++++++++++++++++++ plugins/wasm/Cargo.toml | 2 +- plugins/wasm/README.md | 10 + plugins/wasm/ai-gateway-queue/Cargo.toml | 12 + plugins/wasm/ai-gateway-queue/README.md | 194 ++++++ plugins/wasm/ai-gateway-queue/plugin.yaml | 18 + plugins/wasm/ai-gateway-queue/src/lib.rs | 301 +++++++++ .../plugin/wasm.ai-gateway-queue.json | 29 + 11 files changed, 1217 insertions(+), 1 deletion(-) create mode 100644 binary/ai-gateway-service/Cargo.toml create mode 100644 binary/ai-gateway-service/README.md create mode 100644 binary/ai-gateway-service/src/main.rs create mode 100644 plugins/wasm/ai-gateway-queue/Cargo.toml create mode 100644 plugins/wasm/ai-gateway-queue/README.md create mode 100644 plugins/wasm/ai-gateway-queue/plugin.yaml create mode 100644 plugins/wasm/ai-gateway-queue/src/lib.rs create mode 100644 resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json diff --git a/Cargo.toml b/Cargo.toml index 8b6d228d..d507202b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "binary/spacegate", "binary/admin-server", + "binary/ai-gateway-service", "crates/extension/*", "crates/kernel", "crates/plugin", diff --git a/binary/ai-gateway-service/Cargo.toml b/binary/ai-gateway-service/Cargo.toml new file mode 100644 index 00000000..00fbae5b --- /dev/null +++ b/binary/ai-gateway-service/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "ai-gateway-service" +version.workspace = true +authors.workspace = true +description = "External rate-limit and queue service for SpaceGate AI gateway wasm plugins" +keywords.workspace = true +categories.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +license.workspace = true +edition.workspace = true +readme = "../../README.md" + +[dependencies] +axum = { workspace = true, features = ["tracing", "macros"] } +base64 = { workspace = true } +bytes = { workspace = true } +clap = { version = "4.5", features = ["derive", "env"] } +futures-util = { workspace = true } +fred = { version = "10.1.0", default-features = false, features = ["enable-rustls", "i-keys", "i-scripts", "i-streams", "subscriber-client", "transactions"] } +http = "1" +reqwest = { workspace = true, features = ["json"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +tokio = { workspace = true, features = ["full"] } +tower-http = { version = "0.6", features = ["trace"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } diff --git a/binary/ai-gateway-service/README.md b/binary/ai-gateway-service/README.md new file mode 100644 index 00000000..ea6e2fa5 --- /dev/null +++ b/binary/ai-gateway-service/README.md @@ -0,0 +1,41 @@ +# AI Gateway Service + +External Redis-backed service used by the `ai-gateway-queue` Proxy-Wasm plugin. + +It keeps Redis, worker execution, Pub/Sub waiting, callback delivery, and result storage outside the wasm sandbox. + +## Endpoints + +- `POST /v1/ratelimit/check` + - Reads `X-Tenant-Id`, `X-Model`, and `X-Original-Path`. + - Runs a Redis Lua token bucket. + - Returns `{ "allowed": bool, "retry_after_ms": number }`. +- `POST /v1/queue/enqueue` + - Stores the raw request body and selected headers in Redis Stream. + - Returns `202 Accepted` with `X-Job-Id`. +- `POST /v1/queue/enqueue-and-wait` + - Enqueues the job and waits for the worker result via Redis Pub/Sub. + - Returns the upstream response or `504`. +- `GET /v1/jobs/{job_id}` + - Returns the stored result JSON while the result key TTL is alive. + +## Run + +```bash +cargo run -p ai-gateway-service -- \ + --redis-url redis://127.0.0.1/ \ + --upstream-base-url http://127.0.0.1:9000 +``` + +Useful environment variables: + +```bash +REDIS_URL=redis://127.0.0.1/ +AI_UPSTREAM_BASE_URL=http://127.0.0.1:9000 +AI_RATE_LIMIT_RPS=100 +AI_RATE_LIMIT_BURST=200 +AI_WAIT_TIMEOUT_SECS=60 +AI_WORKER_CONCURRENCY=4 +AI_MAX_BODY_BYTES=33554432 +``` + diff --git a/binary/ai-gateway-service/src/main.rs b/binary/ai-gateway-service/src/main.rs new file mode 100644 index 00000000..9441a2b1 --- /dev/null +++ b/binary/ai-gateway-service/src/main.rs @@ -0,0 +1,581 @@ +use std::collections::HashMap; +use std::net::{IpAddr, SocketAddr}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use axum::body::Bytes; +use axum::extract::{DefaultBodyLimit, Path, State}; +use axum::http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode, Uri}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use base64::Engine; +use clap::Parser; +use fred::clients::{Client as FredClient, SubscriberClient}; +use fred::prelude::*; +use fred::types::streams::XReadResponse; +use serde::{Deserialize, Serialize}; +use tower_http::trace::TraceLayer; + +static JOB_COUNTER: AtomicU64 = AtomicU64::new(1); + +const TOKEN_BUCKET_LUA: &str = r#" +local tokens_key = KEYS[1] +local ts_key = KEYS[2] +local rate = tonumber(ARGV[1]) +local burst = tonumber(ARGV[2]) +local now = tonumber(ARGV[3]) +local cost = tonumber(ARGV[4]) + +if rate <= 0 or burst <= 0 or cost <= 0 then + return {0, 0, 1000} +end + +local burst_milli = burst * 1000 +local cost_milli = cost * 1000 +local tokens = tonumber(redis.call('GET', tokens_key) or burst_milli) +local last_ts = tonumber(redis.call('GET', ts_key) or now) +local elapsed = math.max(0, now - last_ts) +tokens = math.min(burst_milli, tokens + elapsed * rate) + +local ttl = math.max(1000, math.ceil((burst_milli / rate) * 2)) +if tokens >= cost_milli then + tokens = tokens - cost_milli + redis.call('SET', tokens_key, tokens, 'PX', ttl) + redis.call('SET', ts_key, now, 'PX', ttl) + return {1, tokens, 0} +else + local wait_ms = math.ceil((cost_milli - tokens) / rate) + redis.call('SET', tokens_key, tokens, 'PX', ttl) + redis.call('SET', ts_key, now, 'PX', ttl) + return {0, tokens, wait_ms} +end +"#; + +#[derive(Debug, Clone, Parser)] +#[command(version, about = "External Redis-backed rate-limit and queue service for SpaceGate AI gateway")] +struct Args { + #[arg(long, env = "AI_GATEWAY_SERVICE_HOST", default_value = "0.0.0.0")] + host: IpAddr, + #[arg(long, env = "AI_GATEWAY_SERVICE_PORT", default_value_t = 18080)] + port: u16, + #[arg(long, env = "REDIS_URL", default_value = "redis://127.0.0.1/")] + redis_url: String, + #[arg(long, env = "AI_QUEUE_STREAM", default_value = "ai:jobs")] + stream_key: String, + #[arg(long, env = "AI_QUEUE_GROUP", default_value = "ai-gateway-workers")] + consumer_group: String, + #[arg(long, env = "AI_QUEUE_CONSUMER", default_value = "ai-gateway-service")] + consumer_name: String, + #[arg(long, env = "AI_RESULT_KEY_PREFIX", default_value = "result:")] + result_key_prefix: String, + #[arg(long, env = "AI_RESULT_CHANNEL_PREFIX", default_value = "result:")] + result_channel_prefix: String, + #[arg(long, env = "AI_RESULT_TTL_SECS", default_value_t = 120)] + result_ttl_secs: u64, + #[arg(long, env = "AI_RATE_LIMIT_RPS", default_value_t = 100)] + rate_limit_rps: u64, + #[arg(long, env = "AI_RATE_LIMIT_BURST", default_value_t = 200)] + rate_limit_burst: u64, + #[arg(long, env = "AI_WAIT_TIMEOUT_SECS", default_value_t = 60)] + wait_timeout_secs: u64, + #[arg(long, env = "AI_WORKER_CONCURRENCY", default_value_t = 1)] + worker_concurrency: usize, + #[arg(long, env = "AI_UPSTREAM_BASE_URL")] + upstream_base_url: Option, + #[arg(long, env = "AI_MAX_BODY_BYTES", default_value_t = 32 * 1024 * 1024)] + max_body_bytes: usize, +} + +#[derive(Clone)] +struct AppState { + redis: FredClient, + http: reqwest::Client, + cfg: Arc, +} + +#[derive(Debug, Serialize)] +struct RateLimitResponse { + allowed: bool, + remaining_tokens_milli: i64, + retry_after_ms: i64, +} + +#[derive(Debug, Serialize)] +struct EnqueueResponse { + job_id: String, + stream_id: String, + status: &'static str, + poll_url: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +struct StoredResult { + job_id: String, + status: String, + http_status: u16, + headers: HashMap, + body_base64: String, + completed_at_ms: u64, + error: Option, +} + +#[derive(Debug)] +struct ServiceError { + status: StatusCode, + message: String, +} + +impl ServiceError { + fn bad_request(message: impl Into) -> Self { + Self { + status: StatusCode::BAD_REQUEST, + message: message.into(), + } + } + + fn internal(message: impl Into) -> Self { + Self { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: message.into(), + } + } + + fn gateway_timeout(message: impl Into) -> Self { + Self { + status: StatusCode::GATEWAY_TIMEOUT, + message: message.into(), + } + } +} + +impl IntoResponse for ServiceError { + fn into_response(self) -> Response { + let body = Json(serde_json::json!({ "error": self.message })); + (self.status, body).into_response() + } +} + +impl std::fmt::Display for ServiceError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} + +impl std::error::Error for ServiceError {} + +impl From for ServiceError { + fn from(value: fred::error::Error) -> Self { + Self::internal(format!("redis: {value}")) + } +} + +impl From for ServiceError { + fn from(value: reqwest::Error) -> Self { + Self::internal(format!("http: {value}")) + } +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + tracing_subscriber::fmt().with_env_filter(tracing_subscriber::EnvFilter::from_default_env()).init(); + + let args = Args::parse(); + let redis = build_redis_client(&args.redis_url)?; + let _redis_task = redis.init().await?; + let state = AppState { + redis, + http: reqwest::Client::new(), + cfg: Arc::new(args.clone()), + }; + + ensure_consumer_group(&state).await?; + if state.cfg.upstream_base_url.is_some() { + spawn_workers(state.clone()); + } else { + tracing::warn!("AI_UPSTREAM_BASE_URL is not set; queue jobs will be stored but no local worker will process them"); + } + + let app = Router::new() + .route("/healthz", get(healthz)) + .route("/v1/ratelimit/check", post(check_rate_limit)) + .route("/v1/queue/enqueue", post(enqueue)) + .route("/v1/queue/enqueue-and-wait", post(enqueue_and_wait)) + .route("/v1/jobs/{job_id}", get(get_job)) + .layer(DefaultBodyLimit::max(args.max_body_bytes)) + .layer(TraceLayer::new_for_http()) + .with_state(state); + + let addr = SocketAddr::new(args.host, args.port); + tracing::info!(%addr, "ai-gateway-service listening"); + let listener = tokio::net::TcpListener::bind(addr).await?; + axum::serve(listener, app).await?; + Ok(()) +} + +async fn healthz() -> &'static str { + "ok" +} + +async fn check_rate_limit(State(state): State, headers: HeaderMap, uri: Uri) -> Result, ServiceError> { + let tenant = required_header(&headers, "x-tenant-id")?; + let model = optional_header(&headers, "x-model").unwrap_or_else(|| "default".to_string()); + let path = optional_header(&headers, "x-original-path").unwrap_or_else(|| uri.path().to_string()); + let key = sanitize_key(&format!("{tenant}:{model}:{path}")); + let tokens_key = format!("ai:ratelimit:{key}:tokens"); + let ts_key = format!("ai:ratelimit:{key}:ts"); + let now = now_ms(); + + let out: Vec = state + .redis + .eval( + TOKEN_BUCKET_LUA, + vec![tokens_key, ts_key], + vec![ + state.cfg.rate_limit_rps.to_string(), + state.cfg.rate_limit_burst.to_string(), + now.to_string(), + "1".to_string(), + ], + ) + .await?; + + Ok(Json(RateLimitResponse { + allowed: out.first().copied().unwrap_or(0) == 1, + remaining_tokens_milli: out.get(1).copied().unwrap_or(0), + retry_after_ms: out.get(2).copied().unwrap_or(0), + })) +} + +async fn enqueue(State(state): State, method: Method, uri: Uri, headers: HeaderMap, body: Bytes) -> Result { + let accepted = enqueue_job(&state, method, uri, headers, body).await?; + let mut resp = (StatusCode::ACCEPTED, Json(&accepted)).into_response(); + resp.headers_mut().insert("x-job-id", header_value(&accepted.job_id)?); + Ok(resp) +} + +async fn enqueue_and_wait(State(state): State, method: Method, uri: Uri, headers: HeaderMap, body: Bytes) -> Result { + let timeout_secs = optional_header(&headers, "x-request-timeout").and_then(|v| v.parse::().ok()).unwrap_or(state.cfg.wait_timeout_secs); + let accepted = enqueue_job(&state, method, uri, headers, body).await?; + let channel = result_channel(&state, &accepted.job_id); + let subscriber = build_subscriber_client(&state.cfg.redis_url)?; + let _subscriber_task = subscriber.init().await?; + subscriber.subscribe(channel.as_str()).await?; + + if let Some(result) = load_result(&state, &accepted.job_id).await? { + let _ = subscriber.quit().await; + return Ok(result_to_response(result)?); + } + + let mut messages = subscriber.message_rx(); + let wait = tokio::time::timeout(Duration::from_secs(timeout_secs), async { + loop { + let message = messages.recv().await.map_err(|e| ServiceError::internal(format!("pubsub receive: {e}")))?; + if &*message.channel == channel.as_str() { + return Ok::<(), ServiceError>(()); + } + } + }) + .await; + match wait { + Ok(Ok(())) => { + let _ = subscriber.quit().await; + if let Some(result) = load_result(&state, &accepted.job_id).await? { + Ok(result_to_response(result)?) + } else { + Err(ServiceError::gateway_timeout(format!( + "job {} completed notification received but result is missing", + accepted.job_id + ))) + } + } + _ => { + let _ = subscriber.quit().await; + let body = Json(serde_json::json!({ + "error": "timeout", + "job_id": accepted.job_id, + "message": "Job is still processing. Switch to queue mode with a callback for long tasks." + })); + Ok((StatusCode::GATEWAY_TIMEOUT, body).into_response()) + } + } +} + +async fn get_job(State(state): State, Path(job_id): Path) -> Result { + match load_result(&state, &job_id).await? { + Some(result) => Ok(Json(result).into_response()), + None => Ok((StatusCode::NOT_FOUND, Json(serde_json::json!({ "error": "not_found", "job_id": job_id }))).into_response()), + } +} + +async fn enqueue_job(state: &AppState, _method: Method, uri: Uri, headers: HeaderMap, body: Bytes) -> Result { + let job_id = new_job_id(); + let tenant_id = required_header(&headers, "x-tenant-id")?; + let policy = optional_header(&headers, "x-ratelimit-policy").unwrap_or_else(|| "queue".to_string()); + let model = optional_header(&headers, "x-model").unwrap_or_else(|| "default".to_string()); + let callback_url = optional_header(&headers, "x-callback-url").unwrap_or_default(); + let original_method = optional_header(&headers, "x-original-method").unwrap_or_else(|| "POST".to_string()); + let original_path = optional_header(&headers, "x-original-path").unwrap_or_else(|| uri.path().to_string()); + let request_headers = headers_to_json(&headers)?; + let created_at = now_ms(); + + let stream_id: String = state + .redis + .xadd( + state.cfg.stream_key.as_str(), + false, + None::<()>, + "*", + vec![ + ("job_id", Value::String(job_id.clone().into())), + ("tenant_id", Value::String(tenant_id.into())), + ("policy", Value::String(policy.into())), + ("model", Value::String(model.into())), + ("method", Value::String(original_method.into())), + ("path", Value::String(original_path.into())), + ("headers", Value::String(request_headers.into())), + ("body", Value::Bytes(body)), + ("callback_url", Value::String(callback_url.into())), + ("created_at", Value::Integer(created_at as i64)), + ], + ) + .await?; + + Ok(EnqueueResponse { + job_id: job_id.clone(), + stream_id, + status: "queued", + poll_url: format!("/v1/jobs/{job_id}"), + }) +} + +fn spawn_workers(state: AppState) { + for idx in 0..state.cfg.worker_concurrency.max(1) { + let state = state.clone(); + tokio::spawn(async move { + let consumer = format!("{}-{idx}", state.cfg.consumer_name); + loop { + if let Err(e) = worker_once(&state, &consumer).await { + tracing::warn!(error = %e.message, "worker loop failed"); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + }); + } +} + +async fn worker_once(state: &AppState, consumer: &str) -> Result<(), ServiceError> { + let reply: XReadResponse = state + .redis + .xreadgroup_map( + state.cfg.consumer_group.as_str(), + consumer, + Some(5), + Some(1000), + false, + vec![state.cfg.stream_key.as_str()], + vec![">"], + ) + .await?; + + for (_stream, entries) in reply { + for (entry_id, fields) in entries { + match process_job(state, entry_id.as_str(), &fields).await { + Ok(()) => { + let _: i64 = state.redis.xack(state.cfg.stream_key.as_str(), state.cfg.consumer_group.as_str(), vec![entry_id.clone()]).await?; + } + Err(e) => { + tracing::warn!(stream_id = %entry_id, error = %e.message, "job processing failed"); + } + } + } + } + Ok(()) +} + +async fn process_job(state: &AppState, _stream_id: &str, fields: &HashMap) -> Result<(), ServiceError> { + let Some(base) = state.cfg.upstream_base_url.as_deref() else { + return Err(ServiceError::internal("upstream base URL is not configured")); + }; + let job_id = field_string(fields, "job_id").ok_or_else(|| ServiceError::bad_request("job missing job_id"))?; + let method = field_string(fields, "method").unwrap_or_else(|| "POST".to_string()); + let path = field_string(fields, "path").unwrap_or_else(|| "/".to_string()); + let headers_json = field_string(fields, "headers").unwrap_or_else(|| "{}".to_string()); + let callback_url = field_string(fields, "callback_url").unwrap_or_default(); + let body = field_bytes(fields, "body").unwrap_or_default(); + let headers: HashMap = serde_json::from_str(&headers_json).unwrap_or_default(); + + let url = format!("{}{}", base.trim_end_matches('/'), path); + let parsed_method = method.parse::().unwrap_or(reqwest::Method::POST); + let mut req = state.http.request(parsed_method, url); + for (name, value) in headers { + if should_forward_header(&name) { + req = req.header(name, value); + } + } + let upstream = req.body(body).send().await; + let result = match upstream { + Ok(resp) => { + let status = resp.status().as_u16(); + let mut headers = HashMap::new(); + for (name, value) in resp.headers() { + if let Ok(value) = value.to_str() { + headers.insert(name.as_str().to_string(), value.to_string()); + } + } + let body = resp.bytes().await.unwrap_or_default(); + StoredResult { + job_id: job_id.clone(), + status: "completed".to_string(), + http_status: status, + headers, + body_base64: base64::engine::general_purpose::STANDARD.encode(body), + completed_at_ms: now_ms(), + error: None, + } + } + Err(e) => StoredResult { + job_id: job_id.clone(), + status: "failed".to_string(), + http_status: 502, + headers: HashMap::new(), + body_base64: String::new(), + completed_at_ms: now_ms(), + error: Some(e.to_string()), + }, + }; + + store_result(state, &result).await?; + if !callback_url.is_empty() { + let callback_body = serde_json::json!({ + "job_id": result.job_id, + "status": result.status, + "http_status": result.http_status, + "headers": result.headers, + "body_base64": result.body_base64, + "completed_at_ms": result.completed_at_ms, + "error": result.error, + }); + if let Err(e) = state.http.post(callback_url).json(&callback_body).send().await { + tracing::warn!(job_id = %job_id, error = %e, "callback failed"); + } + } + Ok(()) +} + +async fn ensure_consumer_group(state: &AppState) -> Result<(), ServiceError> { + let res: FredResult = state.redis.xgroup_create(state.cfg.stream_key.as_str(), state.cfg.consumer_group.as_str(), "$", true).await; + match res { + Ok(_) => Ok(()), + Err(e) if e.to_string().contains("BUSYGROUP") => Ok(()), + Err(e) => Err(e.into()), + } +} + +async fn store_result(state: &AppState, result: &StoredResult) -> Result<(), ServiceError> { + let json = serde_json::to_string(result).map_err(|e| ServiceError::internal(format!("serialize result: {e}")))?; + let key = result_key(state, &result.job_id); + let channel = result_channel(state, &result.job_id); + let ttl = state.cfg.result_ttl_secs.min(i64::MAX as u64) as i64; + let _: () = state.redis.set(key, json, Some(Expiration::EX(ttl)), None::, false).await?; + let _: i64 = state.redis.publish(channel, "done").await?; + Ok(()) +} + +async fn load_result(state: &AppState, job_id: &str) -> Result, ServiceError> { + let raw: Option = state.redis.get(result_key(state, job_id)).await?; + raw.map(|s| serde_json::from_str(&s).map_err(|e| ServiceError::internal(format!("parse result: {e}")))).transpose() +} + +fn result_to_response(result: StoredResult) -> Result { + let status = StatusCode::from_u16(result.http_status).unwrap_or(StatusCode::OK); + let body = base64::engine::general_purpose::STANDARD.decode(result.body_base64).map_err(|e| ServiceError::internal(format!("decode result body: {e}")))?; + let mut resp = (status, body).into_response(); + for (name, value) in result.headers { + if let (Ok(name), Ok(value)) = (HeaderName::try_from(name.as_str()), HeaderValue::from_str(&value)) { + resp.headers_mut().insert(name, value); + } + } + resp.headers_mut().insert("x-job-id", header_value(&result.job_id)?); + Ok(resp) +} + +fn required_header(headers: &HeaderMap, name: &str) -> Result { + optional_header(headers, name).ok_or_else(|| ServiceError::bad_request(format!("missing required header `{name}`"))) +} + +fn optional_header(headers: &HeaderMap, name: &str) -> Option { + headers.get(name).and_then(|value| value.to_str().ok()).map(str::trim).filter(|value| !value.is_empty()).map(ToOwned::to_owned) +} + +fn headers_to_json(headers: &HeaderMap) -> Result { + let mut out = HashMap::new(); + for (name, value) in headers { + if let Ok(value) = value.to_str() { + out.insert(name.as_str().to_string(), value.to_string()); + } + } + serde_json::to_string(&out).map_err(|e| ServiceError::internal(format!("serialize headers: {e}"))) +} + +fn should_forward_header(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + !matches!( + name.as_str(), + "host" | "connection" | "content-length" | "transfer-encoding" | "x-original-method" | "x-original-path" | "x-ratelimit-policy" | "x-callback-url" | "x-request-timeout" + ) +} + +fn header_value(value: &str) -> Result { + HeaderValue::from_str(value).map_err(|e| ServiceError::internal(format!("invalid response header value: {e}"))) +} + +fn result_key(state: &AppState, job_id: &str) -> String { + format!("{}{}", state.cfg.result_key_prefix, job_id) +} + +fn result_channel(state: &AppState, job_id: &str) -> String { + format!("{}{}", state.cfg.result_channel_prefix, job_id) +} + +fn new_job_id() -> String { + let now = now_ms(); + let seq = JOB_COUNTER.fetch_add(1, Ordering::Relaxed); + format!("{now:x}{seq:x}") +} + +fn now_ms() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64 +} + +fn sanitize_key(input: &str) -> String { + input.chars().map(|ch| if ch.is_ascii_alphanumeric() || matches!(ch, ':' | '_' | '-' | '.') { ch } else { '_' }).collect() +} + +fn build_redis_client(url: &str) -> Result { + let config = Config::from_url(url)?; + Builder::from_config(config).build() +} + +fn build_subscriber_client(url: &str) -> Result { + let config = Config::from_url(url)?; + Builder::from_config(config).build_subscriber_client() +} + +fn field_string(fields: &HashMap, key: &str) -> Option { + fields.get(key).and_then(|value| match value { + Value::String(value) => Some(value.to_string()), + Value::Bytes(value) => String::from_utf8(value.to_vec()).ok(), + Value::Integer(value) => Some(value.to_string()), + _ => None, + }) +} + +fn field_bytes(fields: &HashMap, key: &str) -> Option> { + fields.get(key).and_then(|value| match value { + Value::Bytes(value) => Some(value.to_vec()), + Value::String(value) => Some(value.as_bytes().to_vec()), + _ => None, + }) +} diff --git a/plugins/wasm/Cargo.toml b/plugins/wasm/Cargo.toml index 4cbc3dfd..81063f30 100644 --- a/plugins/wasm/Cargo.toml +++ b/plugins/wasm/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "hello-world", + "ai-gateway-queue", ] resolver = "2" @@ -18,4 +19,3 @@ opt-level = "z" lto = "fat" strip = true panic = "abort" - diff --git a/plugins/wasm/README.md b/plugins/wasm/README.md index ad24725e..4e9db871 100644 --- a/plugins/wasm/README.md +++ b/plugins/wasm/README.md @@ -11,6 +11,10 @@ plugins/wasm/ Cargo.toml src/lib.rs plugin.yaml + ai-gateway-queue/ + Cargo.toml + src/lib.rs + plugin.yaml ``` Use this directory for plugin source code. Keep compiled `.wasm` files in `resource/wasm/` for local demos, or publish them as OCI artifacts/images for Kubernetes usage. @@ -35,6 +39,12 @@ The output for `hello-world` is: plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_hello_world.wasm ``` +The AI gateway queue plugin output is: + +```text +plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm +``` + If you run commands from inside `plugins/wasm/`, the local `.cargo/config.toml` already sets the wasm target: ```bash diff --git a/plugins/wasm/ai-gateway-queue/Cargo.toml b/plugins/wasm/ai-gateway-queue/Cargo.toml new file mode 100644 index 00000000..1c515529 --- /dev/null +++ b/plugins/wasm/ai-gateway-queue/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "spacegate_plugin_ai_gateway_queue" +version.workspace = true +edition.workspace = true +publish.workspace = true +description = "AI gateway rate-limit and queue Proxy-Wasm plugin for SpaceGate." + +[lib] +crate-type = ["cdylib"] + +[dependencies] +proxy-wasm.workspace = true diff --git a/plugins/wasm/ai-gateway-queue/README.md b/plugins/wasm/ai-gateway-queue/README.md new file mode 100644 index 00000000..49acb33c --- /dev/null +++ b/plugins/wasm/ai-gateway-queue/README.md @@ -0,0 +1,194 @@ +# ai-gateway-queue + +`ai-gateway-queue` 是一个运行在 SpaceGate Wasm 里的 AI 网关插件，用来把请求按策略分成三种处理方式： + +- `abandon`：先做限流检查，失败直接返回 +- `queue`：进入外部队列，立即返回 `202` +- `wait`：进入外部队列并等待结果返回 + +它本身不直接访问 Redis，而是通过 `dispatch_http_call` 调用外部的 `ai-gateway-service`，再由该服务去处理 Redis、队列和等待逻辑。 + +## 架构 + +```text +Client + -> SpaceGate / ai-gateway-queue wasm plugin + -> ai-gateway-service + -> Redis / Worker / Upstream AI Service +``` + +## 依赖 + +- SpaceGate 已启用 Wasm 支持 +- Rust 工具链 +- `wasm32-wasip1` 目标 +- Redis +- `ai-gateway-service` + +安装 wasm 目标： + +```bash +rustup target add wasm32-wasip1 +``` + +## 构建 + +在 `spacegate` 目录下执行： + +```bash +cargo build --release --target wasm32-wasip1 --manifest-path plugins/wasm/Cargo.toml -p spacegate_plugin_ai_gateway_queue +``` + +编译产物： + +```text +plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm +``` + +## 启动外部服务 + +`ai-gateway-queue` 依赖外部服务来完成限流、入队、等待和回调。 + +```bash +cargo run -p ai-gateway-service -- \ + --redis-url redis://127.0.0.1/ \ + --upstream-base-url http://127.0.0.1:9000 +``` + +常用环境变量： + +```bash +REDIS_URL=redis://127.0.0.1/ +AI_UPSTREAM_BASE_URL=http://127.0.0.1:9000 +AI_RATE_LIMIT_RPS=100 +AI_RATE_LIMIT_BURST=200 +AI_WAIT_TIMEOUT_SECS=60 +AI_WORKER_CONCURRENCY=4 +AI_MAX_BODY_BYTES=33554432 +``` + +如果不设置 `AI_UPSTREAM_BASE_URL`，队列任务仍会写入 Redis，但不会由本地 worker 消费。 + +## SpaceGate 配置 + +可参考： + +`/Users/sh.zhang/Workspace/huayun/jiyan/ai-gateway-dev/spacegate/resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json` + +关键配置项： + +```json +{ + "url": "plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm", + "fail_strategy": "fail_close", + "plugin_name": "ai-gateway-queue", + "vm_pool_size": 4, + "wait_vm_pool_size": 4, + "limits": { + "max_memory_pages": 64, + "fuel_per_call": 20000000, + "epoch_timeout_millis": 50, + "max_body_bytes": 33554432, + "max_pending_calls": 1 + }, + "plugin_config": { + "service_cluster": "ai-gateway-service", + "service_authority": "ai-gateway-service", + "rate_limit_path": "/v1/ratelimit/check", + "enqueue_path": "/v1/queue/enqueue", + "wait_path": "/v1/queue/enqueue-and-wait", + "service_timeout_ms": 65000, + "require_policy": true + }, + "clusters": { + "ai-gateway-service": "http://127.0.0.1:18080" + } +} +``` + +### `plugin_config` 说明 + +- `service_cluster`：外部服务所在 cluster 名称 +- `service_authority`：转发时使用的 `:authority` +- `rate_limit_path`：限流检查接口 +- `enqueue_path`：入队接口 +- `wait_path`：入队并等待接口 +- `service_timeout_ms`：调用外部服务超时 +- `require_policy`：是否强制要求请求头携带策略 + +## 请求头 + +插件依赖下列请求头： + +- `X-RateLimit-Policy`：必填，取值为 `abandon`、`queue`、`wait` +- `X-Tenant-Id`：必填 +- `X-Callback-URL`：`queue` 场景下建议提供 +- `X-Request-Timeout`：`wait` 场景下可选，单位为秒 +- `X-Model`：可选，透传给外部服务 + +Header 名称大小写不敏感；`X-RateLimit-Policy` 的值请使用小写。 + +## 三种模式 + +### 1. `abandon` + +先调用限流接口，允许则继续转发到后端，拒绝则返回 `429`。 + +示例： + +```bash +curl -i http://localhost:9080/your/api \ + -H 'X-RateLimit-Policy: abandon' \ + -H 'X-Tenant-Id: demo' \ + -H 'X-Model: gpt-4o-mini' \ + -d '{"prompt":"hello"}' +``` + +### 2. `queue` + +请求体进入队列，插件立即返回 `202 Accepted`，响应里会带 `X-Job-Id`。 + +示例： + +```bash +curl -i http://localhost:9080/your/api \ + -H 'X-RateLimit-Policy: queue' \ + -H 'X-Tenant-Id: demo' \ + -H 'X-Callback-URL: http://localhost:9001/callback' \ + -d '{"prompt":"hello"}' +``` + +### 3. `wait` + +请求体进入队列后等待结果返回。成功时直接返回上游响应，超时则返回 `504`。 + +示例： + +```bash +curl -i http://localhost:9080/your/api \ + -H 'X-RateLimit-Policy: wait' \ + -H 'X-Tenant-Id: demo' \ + -H 'X-Request-Timeout: 60' \ + -d '{"prompt":"hello"}' +``` + +## 返回行为 + +- `400`：缺少必要请求头或策略非法 +- `429`：限流拒绝 +- `202`：队列已接收 +- `200`/`4xx`/`5xx`：`wait` 模式下，由外部服务返回 +- `502`：外部服务不可达或调用失败 + +## 调试建议 + +- 先确认 `ai-gateway-service` 已启动并能连上 Redis +- 再确认 SpaceGate 的 `clusters.ai-gateway-service` 指向正确地址 +- `wait` 模式建议单独使用 `wait_vm_pool_size`，避免拖垮普通请求 +- 如果请求一直返回 `400 missing_or_invalid_rate_limit_policy`，先检查 `X-RateLimit-Policy` + +## 备注 + +- 这个插件当前是面向 OpenAI 风格 AI 请求的队列和限流入口 +- Redis 相关逻辑被放在 wasm 外部服务中，便于隔离和演进 +- 具体协议和接口字段，以 `ai-gateway-service` 的实现为准 diff --git a/plugins/wasm/ai-gateway-queue/plugin.yaml b/plugins/wasm/ai-gateway-queue/plugin.yaml new file mode 100644 index 00000000..6404f60e --- /dev/null +++ b/plugins/wasm/ai-gateway-queue/plugin.yaml @@ -0,0 +1,18 @@ +apiVersion: extensions.higress.io/v1alpha1 +kind: WasmPlugin +metadata: + name: ai-gateway-queue + namespace: spacegate +spec: + url: oci://registry.example.com/spacegate/plugins/ai-gateway-queue:v1 + pluginName: ai-gateway-queue + phase: AUTHN + priority: 90 + failStrategy: FAIL_CLOSE + defaultConfig: + service_cluster: ai-gateway-service + service_authority: ai-gateway-service + rate_limit_path: /v1/ratelimit/check + enqueue_path: /v1/queue/enqueue + wait_path: /v1/queue/enqueue-and-wait + service_timeout_ms: 65000 diff --git a/plugins/wasm/ai-gateway-queue/src/lib.rs b/plugins/wasm/ai-gateway-queue/src/lib.rs new file mode 100644 index 00000000..4b9fd369 --- /dev/null +++ b/plugins/wasm/ai-gateway-queue/src/lib.rs @@ -0,0 +1,301 @@ +use std::time::Duration; + +use proxy_wasm::hostcalls; +use proxy_wasm::traits::*; +use proxy_wasm::types::*; + +proxy_wasm::main! {{ + proxy_wasm::set_log_level(LogLevel::Info); + proxy_wasm::set_root_context(|_| -> Box { Box::new(AiGatewayRoot::default()) }); +}} + +#[derive(Clone)] +struct AiGatewayConfig { + service_cluster: String, + service_authority: String, + rate_limit_path: String, + enqueue_path: String, + wait_path: String, + service_timeout_ms: u64, + require_policy: bool, +} + +impl Default for AiGatewayConfig { + fn default() -> Self { + Self { + service_cluster: "ai-gateway-service".to_string(), + service_authority: "ai-gateway-service".to_string(), + rate_limit_path: "/v1/ratelimit/check".to_string(), + enqueue_path: "/v1/queue/enqueue".to_string(), + wait_path: "/v1/queue/enqueue-and-wait".to_string(), + service_timeout_ms: 65_000, + require_policy: true, + } + } +} + +#[derive(Default)] +struct AiGatewayRoot { + cfg: AiGatewayConfig, +} + +impl Context for AiGatewayRoot {} + +impl RootContext for AiGatewayRoot { + fn on_vm_start(&mut self, _: usize) -> bool { + let _ = hostcalls::log(LogLevel::Info, "ai-gateway-queue wasm plugin started"); + true + } + + fn on_configure(&mut self, _: usize) -> bool { + let raw = self.get_plugin_configuration().unwrap_or_default(); + let text = String::from_utf8_lossy(&raw); + let mut cfg = AiGatewayConfig::default(); + for line in text.lines() { + let Some((key, value)) = line.split_once(':') else { continue }; + let value = value.trim().trim_matches(['"', '\''].as_ref()); + match key.trim() { + "service_cluster" => cfg.service_cluster = value.to_string(), + "service_authority" => cfg.service_authority = value.to_string(), + "rate_limit_path" => cfg.rate_limit_path = value.to_string(), + "enqueue_path" => cfg.enqueue_path = value.to_string(), + "wait_path" => cfg.wait_path = value.to_string(), + "service_timeout_ms" => cfg.service_timeout_ms = value.parse().unwrap_or(cfg.service_timeout_ms), + "require_policy" => cfg.require_policy = value.parse().unwrap_or(true), + _ => {} + } + } + self.cfg = cfg; + true + } + + fn create_http_context(&self, _: u32) -> Option> { + Some(Box::new(AiGatewayHttp { + cfg: self.cfg.clone(), + pending: None, + deferred_policy: None, + })) + } + + fn get_type(&self) -> Option { + Some(ContextType::HttpContext) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Policy { + Abandon, + Queue, + Wait, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Pending { + RateLimit, + Queue, + Wait, +} + +struct AiGatewayHttp { + cfg: AiGatewayConfig, + pending: Option<(u32, Pending)>, + deferred_policy: Option, +} + +impl Context for AiGatewayHttp { + fn on_http_call_response(&mut self, token_id: u32, _num_headers: usize, body_size: usize, _num_trailers: usize) { + let Some((pending_token, pending)) = self.pending else { + return; + }; + if token_id != pending_token { + return; + } + self.pending = None; + + match pending { + Pending::RateLimit => self.handle_rate_limit_response(body_size), + Pending::Queue | Pending::Wait => self.forward_service_response(body_size), + } + } +} + +impl HttpContext for AiGatewayHttp { + fn on_http_request_headers(&mut self, _: usize, end_of_stream: bool) -> Action { + let Some(policy) = self.request_policy() else { + if self.cfg.require_policy { + self.send_json(400, r#"{"error":"missing_or_invalid_rate_limit_policy"}"#); + return Action::Pause; + } + return Action::Continue; + }; + + if self.get_http_request_header("x-tenant-id").is_none() { + self.send_json(400, r#"{"error":"missing_x_tenant_id"}"#); + return Action::Pause; + } + + match policy { + Policy::Abandon => { + if self.dispatch_service_call(Pending::RateLimit, &self.cfg.rate_limit_path.clone(), None) { + Action::Pause + } else { + self.send_json(502, r#"{"error":"rate_limit_service_unavailable"}"#); + Action::Pause + } + } + Policy::Queue | Policy::Wait => { + if end_of_stream { + let pending = if policy == Policy::Queue { Pending::Queue } else { Pending::Wait }; + let path = if policy == Policy::Queue { + self.cfg.enqueue_path.clone() + } else { + self.cfg.wait_path.clone() + }; + if !self.dispatch_service_call(pending, &path, Some(&[])) { + self.send_json(502, r#"{"error":"queue_service_unavailable"}"#); + } + } else { + self.deferred_policy = Some(policy); + } + Action::Pause + } + } + } + + fn on_http_request_body(&mut self, body_size: usize, end_of_stream: bool) -> Action { + let Some(policy) = self.deferred_policy else { + return Action::Continue; + }; + if !end_of_stream { + return Action::Pause; + } + self.deferred_policy = None; + let body = self.get_http_request_body(0, body_size).unwrap_or_default(); + let pending = if policy == Policy::Queue { Pending::Queue } else { Pending::Wait }; + let path = if policy == Policy::Queue { + self.cfg.enqueue_path.clone() + } else { + self.cfg.wait_path.clone() + }; + if !self.dispatch_service_call(pending, &path, Some(&body)) { + self.send_json(502, r#"{"error":"queue_service_unavailable"}"#); + } + Action::Pause + } +} + +impl AiGatewayHttp { + fn request_policy(&self) -> Option { + match self.get_http_request_header("x-ratelimit-policy").as_deref().map(str::trim) { + Some("abandon") => Some(Policy::Abandon), + Some("queue") => Some(Policy::Queue), + Some("wait") => Some(Policy::Wait), + _ => None, + } + } + + fn dispatch_service_call(&mut self, pending: Pending, path: &str, body: Option<&[u8]>) -> bool { + let headers = self.service_headers(path); + let refs = headers.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect::>(); + match self.dispatch_http_call(&self.cfg.service_cluster, refs, body, vec![], Duration::from_millis(self.cfg.service_timeout_ms)) { + Ok(token) => { + self.pending = Some((token, pending)); + true + } + Err(status) => { + let _ = hostcalls::log(LogLevel::Warn, &format!("dispatch service call failed: {status:?}")); + false + } + } + } + + fn service_headers(&self, path: &str) -> Vec<(String, String)> { + let mut out = vec![ + (":method".to_string(), "POST".to_string()), + (":path".to_string(), path.to_string()), + (":authority".to_string(), self.cfg.service_authority.clone()), + ( + "x-original-method".to_string(), + self.get_http_request_header(":method").unwrap_or_else(|| "POST".to_string()), + ), + ("x-original-path".to_string(), self.get_http_request_header(":path").unwrap_or_else(|| "/".to_string())), + ]; + + for (name, value) in self.get_http_request_headers() { + if should_forward_to_service(&name) { + out.push((name, value)); + } + } + out + } + + fn handle_rate_limit_response(&mut self, body_size: usize) { + let status = self.service_status(); + let body = self.get_http_call_response_body(0, body_size).unwrap_or_default(); + let text = String::from_utf8_lossy(&body); + if status == 200 && contains_allowed_true(&text) { + self.resume_http_request(); + return; + } + if status == 200 { + let retry_after_ms = extract_json_number(&text, "retry_after_ms").unwrap_or(1000); + let retry_after_secs = ((retry_after_ms + 999) / 1000).max(1).to_string(); + let retry_after_ms = retry_after_ms.to_string(); + let headers = [ + ("content-type".to_string(), "application/json".to_string()), + ("retry-after".to_string(), retry_after_secs), + ("x-ratelimit-retry-after-ms".to_string(), retry_after_ms), + ]; + let headers = headers.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect::>(); + self.send_http_response(429, headers, Some(text.as_bytes())); + } else { + self.send_json(502, r#"{"error":"rate_limit_service_error"}"#); + } + } + + fn forward_service_response(&mut self, body_size: usize) { + let status = self.service_status(); + let body = self.get_http_call_response_body(0, body_size).unwrap_or_default(); + let header_storage = self.response_headers_for_client(); + let headers = header_storage.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect::>(); + self.send_http_response(status as u32, headers, Some(&body)); + } + + fn response_headers_for_client(&self) -> Vec<(String, String)> { + let mut out = Vec::new(); + for name in ["content-type", "x-job-id", "retry-after"] { + if let Some(value) = self.get_http_call_response_header(name) { + out.push((name.to_string(), value)); + } + } + out + } + + fn service_status(&self) -> u16 { + self.get_http_call_response_header(":status").and_then(|v| v.parse().ok()).unwrap_or(502) + } + + fn send_json(&self, status: u32, body: &str) { + self.send_http_response(status, vec![("content-type", "application/json")], Some(body.as_bytes())); + } +} + +fn should_forward_to_service(name: &str) -> bool { + let lower = name.to_ascii_lowercase(); + !lower.starts_with(':') + && !matches!( + lower.as_str(), + "host" | "connection" | "content-length" | "transfer-encoding" | "x-original-method" | "x-original-path" + ) +} + +fn contains_allowed_true(text: &str) -> bool { + text.contains(r#""allowed":true"#) || text.contains(r#""allowed": true"#) +} + +fn extract_json_number(text: &str, key: &str) -> Option { + let needle = format!(r#""{key}":"#); + let pos = text.find(&needle)?; + let digits = text[pos + needle.len()..].chars().skip_while(|c| c.is_whitespace()).take_while(|c| c.is_ascii_digit()).collect::(); + digits.parse().ok() +} diff --git a/resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json b/resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json new file mode 100644 index 00000000..4eb0b630 --- /dev/null +++ b/resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json @@ -0,0 +1,29 @@ +{ + "url": "plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm", + "validate_on_create": false, + "fail_strategy": "fail_close", + "plugin_name": "ai-gateway-queue", + "plugin_root_id": "ai-gateway-queue-root", + "plugin_vm_id": "ai-gateway-queue-vm", + "vm_pool_size": 4, + "wait_vm_pool_size": 4, + "limits": { + "max_memory_pages": 64, + "fuel_per_call": 20000000, + "epoch_timeout_millis": 50, + "max_body_bytes": 33554432, + "max_pending_calls": 1 + }, + "plugin_config": { + "service_cluster": "ai-gateway-service", + "service_authority": "ai-gateway-service", + "rate_limit_path": "/v1/ratelimit/check", + "enqueue_path": "/v1/queue/enqueue", + "wait_path": "/v1/queue/enqueue-and-wait", + "service_timeout_ms": 65000, + "require_policy": true + }, + "clusters": { + "ai-gateway-service": "http://127.0.0.1:18080" + } +} From f6fe26de75c260780a7efdf808cec7675ffc1e69 Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Wed, 20 May 2026 20:06:01 +0800 Subject: [PATCH 08/19] feat: harden ai gateway queue service --- binary/ai-gateway-service/README.md | 31 +- binary/ai-gateway-service/src/main.rs | 519 ++++++++++++++++++++--- plugins/wasm/ai-gateway-queue/README.md | 26 +- plugins/wasm/ai-gateway-queue/src/lib.rs | 7 +- 4 files changed, 516 insertions(+), 67 deletions(-) diff --git a/binary/ai-gateway-service/README.md b/binary/ai-gateway-service/README.md index ea6e2fa5..fa5221cc 100644 --- a/binary/ai-gateway-service/README.md +++ b/binary/ai-gateway-service/README.md @@ -9,15 +9,19 @@ It keeps Redis, worker execution, Pub/Sub waiting, callback delivery, and result - `POST /v1/ratelimit/check` - Reads `X-Tenant-Id`, `X-Model`, and `X-Original-Path`. - Runs a Redis Lua token bucket. + - Can override per tenant with Redis keys `ai:tenant:ratelimit:{tenant}:rps` and `ai:tenant:ratelimit:{tenant}:burst`. - Returns `{ "allowed": bool, "retry_after_ms": number }`. - `POST /v1/queue/enqueue` - - Stores the raw request body and selected headers in Redis Stream. + - Requires `X-Callback-URL` by default. + - Stores selected headers and either inline base64 body or an object-store reference in Redis Stream. - Returns `202 Accepted` with `X-Job-Id`. - `POST /v1/queue/enqueue-and-wait` - Enqueues the job and waits for the worker result via Redis Pub/Sub. - - Returns the upstream response or `504`. + - Returns the upstream response with `X-Job-Id` and `X-Queue-Wait-Ms`, or `504`. - `GET /v1/jobs/{job_id}` - Returns the stored result JSON while the result key TTL is alive. +- `GET /metrics` + - Returns Prometheus text metrics for queue depth, limits, callbacks, retries, and worker counters. ## Run @@ -37,5 +41,28 @@ AI_RATE_LIMIT_BURST=200 AI_WAIT_TIMEOUT_SECS=60 AI_WORKER_CONCURRENCY=4 AI_MAX_BODY_BYTES=33554432 +AI_INLINE_THRESHOLD=131072 +AI_QUEUE_MAX_LEN=100000 +AI_RECLAIM_INTERVAL_SECS=30 +AI_RECLAIM_MIN_IDLE_SECS=30 +AI_REQUIRE_HTTPS_CALLBACK=true ``` +Optional object offload variables: + +```bash +AI_OBJECT_STORE_ENDPOINT=http://127.0.0.1:9000 +AI_OBJECT_STORE_BUCKET=ai-gateway-body +AI_OBJECT_STORE_PREFIX=bodies +AI_OBJECT_STORE_AUTH_HEADER='Authorization: Bearer token' +``` + +Priority queues are disabled by default. Enable them and send `X-Queue-Priority: high|low` to route jobs to separate streams: + +```bash +AI_ENABLE_PRIORITY_STREAMS=true +AI_QUEUE_HIGH_STREAM=ai:jobs:high +AI_QUEUE_LOW_STREAM=ai:jobs:low +``` + +Callback failures are written to `AI_CALLBACK_RETRY_STREAM` and retried by a local retry worker. Pending Redis Stream jobs are reclaimed with `XAUTOCLAIM` according to the reclaim settings. diff --git a/binary/ai-gateway-service/src/main.rs b/binary/ai-gateway-service/src/main.rs index 9441a2b1..082034c8 100644 --- a/binary/ai-gateway-service/src/main.rs +++ b/binary/ai-gateway-service/src/main.rs @@ -16,6 +16,7 @@ use fred::clients::{Client as FredClient, SubscriberClient}; use fred::prelude::*; use fred::types::streams::XReadResponse; use serde::{Deserialize, Serialize}; +use tokio::sync::Semaphore; use tower_http::trace::TraceLayer; static JOB_COUNTER: AtomicU64 = AtomicU64::new(1); @@ -64,10 +65,22 @@ struct Args { redis_url: String, #[arg(long, env = "AI_QUEUE_STREAM", default_value = "ai:jobs")] stream_key: String, + #[arg(long, env = "AI_QUEUE_HIGH_STREAM", default_value = "ai:jobs:high")] + high_priority_stream_key: String, + #[arg(long, env = "AI_QUEUE_LOW_STREAM", default_value = "ai:jobs:low")] + low_priority_stream_key: String, + #[arg(long, env = "AI_ENABLE_PRIORITY_STREAMS", default_value_t = false)] + enable_priority_streams: bool, + #[arg(long, env = "AI_QUEUE_MAX_LEN", default_value_t = 100_000)] + stream_max_len: u64, #[arg(long, env = "AI_QUEUE_GROUP", default_value = "ai-gateway-workers")] consumer_group: String, #[arg(long, env = "AI_QUEUE_CONSUMER", default_value = "ai-gateway-service")] consumer_name: String, + #[arg(long, env = "AI_CALLBACK_RETRY_STREAM", default_value = "ai:callback-retry")] + callback_retry_stream: String, + #[arg(long, env = "AI_CALLBACK_RETRY_GROUP", default_value = "ai-gateway-callbacks")] + callback_retry_group: String, #[arg(long, env = "AI_RESULT_KEY_PREFIX", default_value = "result:")] result_key_prefix: String, #[arg(long, env = "AI_RESULT_CHANNEL_PREFIX", default_value = "result:")] @@ -78,6 +91,8 @@ struct Args { rate_limit_rps: u64, #[arg(long, env = "AI_RATE_LIMIT_BURST", default_value_t = 200)] rate_limit_burst: u64, + #[arg(long, env = "AI_TENANT_RATE_LIMIT_PREFIX", default_value = "ai:tenant:ratelimit:")] + tenant_rate_limit_prefix: String, #[arg(long, env = "AI_WAIT_TIMEOUT_SECS", default_value_t = 60)] wait_timeout_secs: u64, #[arg(long, env = "AI_WORKER_CONCURRENCY", default_value_t = 1)] @@ -86,6 +101,24 @@ struct Args { upstream_base_url: Option, #[arg(long, env = "AI_MAX_BODY_BYTES", default_value_t = 32 * 1024 * 1024)] max_body_bytes: usize, + #[arg(long, env = "AI_INLINE_THRESHOLD", default_value_t = 128 * 1024)] + inline_threshold: usize, + #[arg(long, env = "AI_BODY_READ_CONCURRENCY", default_value_t = 200)] + body_read_concurrency: usize, + #[arg(long, env = "AI_RECLAIM_INTERVAL_SECS", default_value_t = 30)] + reclaim_interval_secs: u64, + #[arg(long, env = "AI_RECLAIM_MIN_IDLE_SECS", default_value_t = 30)] + reclaim_min_idle_secs: u64, + #[arg(long, env = "AI_REQUIRE_HTTPS_CALLBACK", default_value_t = true)] + require_https_callback: bool, + #[arg(long, env = "AI_OBJECT_STORE_ENDPOINT")] + object_store_endpoint: Option, + #[arg(long, env = "AI_OBJECT_STORE_BUCKET", default_value = "ai-gateway-body")] + object_store_bucket: String, + #[arg(long, env = "AI_OBJECT_STORE_PREFIX", default_value = "bodies")] + object_store_prefix: String, + #[arg(long, env = "AI_OBJECT_STORE_AUTH_HEADER")] + object_store_auth_header: Option, } #[derive(Clone)] @@ -93,6 +126,24 @@ struct AppState { redis: FredClient, http: reqwest::Client, cfg: Arc, + body_permits: Arc, + metrics: Arc, +} + +#[derive(Default)] +struct Metrics { + rate_limited_total: AtomicU64, + enqueue_total: AtomicU64, + enqueue_queue_total: AtomicU64, + enqueue_wait_total: AtomicU64, + wait_timeout_total: AtomicU64, + callback_failure_total: AtomicU64, + callback_retry_total: AtomicU64, + callback_retry_success_total: AtomicU64, + worker_completed_total: AtomicU64, + worker_failed_total: AtomicU64, + reclaimed_total: AtomicU64, + object_offload_total: AtomicU64, } #[derive(Debug, Serialize)] @@ -106,6 +157,7 @@ struct RateLimitResponse { struct EnqueueResponse { job_id: String, stream_id: String, + stream_key: String, status: &'static str, poll_url: String, } @@ -121,6 +173,35 @@ struct StoredResult { error: Option, } +#[derive(Debug)] +struct AcceptedJob { + response: EnqueueResponse, + created_at_ms: u64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum QueuePolicy { + Queue, + Wait, +} + +impl QueuePolicy { + fn as_str(self) -> &'static str { + match self { + QueuePolicy::Queue => "queue", + QueuePolicy::Wait => "wait", + } + } +} + +#[derive(Debug)] +struct BodyLocation { + body_base64: String, + object_ref: String, + size: usize, + storage: &'static str, +} + #[derive(Debug)] struct ServiceError { status: StatusCode, @@ -188,17 +269,22 @@ async fn main() -> Result<(), Box> { redis, http: reqwest::Client::new(), cfg: Arc::new(args.clone()), + body_permits: Arc::new(Semaphore::new(args.body_read_concurrency.max(1))), + metrics: Arc::new(Metrics::default()), }; - ensure_consumer_group(&state).await?; + ensure_consumer_groups(&state).await?; if state.cfg.upstream_base_url.is_some() { spawn_workers(state.clone()); + spawn_reclaimer(state.clone()); + spawn_callback_retry_worker(state.clone()); } else { tracing::warn!("AI_UPSTREAM_BASE_URL is not set; queue jobs will be stored but no local worker will process them"); } let app = Router::new() .route("/healthz", get(healthz)) + .route("/metrics", get(metrics)) .route("/v1/ratelimit/check", post(check_rate_limit)) .route("/v1/queue/enqueue", post(enqueue)) .route("/v1/queue/enqueue-and-wait", post(enqueue_and_wait)) @@ -222,6 +308,7 @@ async fn check_rate_limit(State(state): State, headers: HeaderMap, uri let tenant = required_header(&headers, "x-tenant-id")?; let model = optional_header(&headers, "x-model").unwrap_or_else(|| "default".to_string()); let path = optional_header(&headers, "x-original-path").unwrap_or_else(|| uri.path().to_string()); + let (rate_limit_rps, rate_limit_burst) = tenant_rate_limit(&state, &tenant).await?; let key = sanitize_key(&format!("{tenant}:{model}:{path}")); let tokens_key = format!("ai:ratelimit:{key}:tokens"); let ts_key = format!("ai:ratelimit:{key}:ts"); @@ -232,40 +319,40 @@ async fn check_rate_limit(State(state): State, headers: HeaderMap, uri .eval( TOKEN_BUCKET_LUA, vec![tokens_key, ts_key], - vec![ - state.cfg.rate_limit_rps.to_string(), - state.cfg.rate_limit_burst.to_string(), - now.to_string(), - "1".to_string(), - ], + vec![rate_limit_rps.to_string(), rate_limit_burst.to_string(), now.to_string(), "1".to_string()], ) .await?; + let allowed = out.first().copied().unwrap_or(0) == 1; + if !allowed { + state.metrics.rate_limited_total.fetch_add(1, Ordering::Relaxed); + } Ok(Json(RateLimitResponse { - allowed: out.first().copied().unwrap_or(0) == 1, + allowed, remaining_tokens_milli: out.get(1).copied().unwrap_or(0), retry_after_ms: out.get(2).copied().unwrap_or(0), })) } async fn enqueue(State(state): State, method: Method, uri: Uri, headers: HeaderMap, body: Bytes) -> Result { - let accepted = enqueue_job(&state, method, uri, headers, body).await?; - let mut resp = (StatusCode::ACCEPTED, Json(&accepted)).into_response(); - resp.headers_mut().insert("x-job-id", header_value(&accepted.job_id)?); + let accepted = enqueue_job(&state, QueuePolicy::Queue, method, uri, headers, body).await?; + let mut resp = (StatusCode::ACCEPTED, Json(&accepted.response)).into_response(); + resp.headers_mut().insert("x-job-id", header_value(&accepted.response.job_id)?); + resp.headers_mut().insert("location", header_value(&accepted.response.poll_url)?); Ok(resp) } async fn enqueue_and_wait(State(state): State, method: Method, uri: Uri, headers: HeaderMap, body: Bytes) -> Result { let timeout_secs = optional_header(&headers, "x-request-timeout").and_then(|v| v.parse::().ok()).unwrap_or(state.cfg.wait_timeout_secs); - let accepted = enqueue_job(&state, method, uri, headers, body).await?; - let channel = result_channel(&state, &accepted.job_id); + let accepted = enqueue_job(&state, QueuePolicy::Wait, method, uri, headers, body).await?; + let channel = result_channel(&state, &accepted.response.job_id); let subscriber = build_subscriber_client(&state.cfg.redis_url)?; let _subscriber_task = subscriber.init().await?; subscriber.subscribe(channel.as_str()).await?; - if let Some(result) = load_result(&state, &accepted.job_id).await? { + if let Some(result) = load_result(&state, &accepted.response.job_id).await? { let _ = subscriber.quit().await; - return Ok(result_to_response(result)?); + return Ok(result_to_response(result, accepted.created_at_ms)?); } let mut messages = subscriber.message_rx(); @@ -281,20 +368,24 @@ async fn enqueue_and_wait(State(state): State, method: Method, uri: Ur match wait { Ok(Ok(())) => { let _ = subscriber.quit().await; - if let Some(result) = load_result(&state, &accepted.job_id).await? { - Ok(result_to_response(result)?) + if let Some(result) = load_result(&state, &accepted.response.job_id).await? { + Ok(result_to_response(result, accepted.created_at_ms)?) } else { Err(ServiceError::gateway_timeout(format!( "job {} completed notification received but result is missing", - accepted.job_id + accepted.response.job_id ))) } } _ => { let _ = subscriber.quit().await; + state.metrics.wait_timeout_total.fetch_add(1, Ordering::Relaxed); + let waited_ms = now_ms().saturating_sub(accepted.created_at_ms); let body = Json(serde_json::json!({ "error": "timeout", - "job_id": accepted.job_id, + "job_id": accepted.response.job_id, + "poll_url": accepted.response.poll_url, + "waited_ms": waited_ms, "message": "Job is still processing. Switch to queue mode with a callback for long tasks." })); Ok((StatusCode::GATEWAY_TIMEOUT, body).into_response()) @@ -309,44 +400,109 @@ async fn get_job(State(state): State, Path(job_id): Path) -> R } } -async fn enqueue_job(state: &AppState, _method: Method, uri: Uri, headers: HeaderMap, body: Bytes) -> Result { +async fn metrics(State(state): State) -> Result { + let queue_depth: i64 = state.redis.xlen(state.cfg.stream_key.as_str()).await.unwrap_or_default(); + let high_queue_depth: i64 = state.redis.xlen(state.cfg.high_priority_stream_key.as_str()).await.unwrap_or_default(); + let low_queue_depth: i64 = state.redis.xlen(state.cfg.low_priority_stream_key.as_str()).await.unwrap_or_default(); + let callback_retry_depth: i64 = state.redis.xlen(state.cfg.callback_retry_stream.as_str()).await.unwrap_or_default(); + + let body = format!( + "\ +rate_limited_total {}\n\ +enqueue_total {}\n\ +enqueue_total{{policy=\"queue\"}} {}\n\ +enqueue_total{{policy=\"wait\"}} {}\n\ +wait_timeout_total {}\n\ +callback_failure_total {}\n\ +callback_retry_total {}\n\ +callback_retry_success_total {}\n\ +worker_completed_total {}\n\ +worker_failed_total {}\n\ +reclaimed_total {}\n\ +object_offload_total {}\n\ +queue_depth {}\n\ +queue_depth{{priority=\"high\"}} {}\n\ +queue_depth{{priority=\"low\"}} {}\n\ +callback_retry_depth {}\n", + state.metrics.rate_limited_total.load(Ordering::Relaxed), + state.metrics.enqueue_total.load(Ordering::Relaxed), + state.metrics.enqueue_queue_total.load(Ordering::Relaxed), + state.metrics.enqueue_wait_total.load(Ordering::Relaxed), + state.metrics.wait_timeout_total.load(Ordering::Relaxed), + state.metrics.callback_failure_total.load(Ordering::Relaxed), + state.metrics.callback_retry_total.load(Ordering::Relaxed), + state.metrics.callback_retry_success_total.load(Ordering::Relaxed), + state.metrics.worker_completed_total.load(Ordering::Relaxed), + state.metrics.worker_failed_total.load(Ordering::Relaxed), + state.metrics.reclaimed_total.load(Ordering::Relaxed), + state.metrics.object_offload_total.load(Ordering::Relaxed), + queue_depth, + high_queue_depth, + low_queue_depth, + callback_retry_depth, + ); + Ok((StatusCode::OK, [("content-type", "text/plain; version=0.0.4")], body).into_response()) +} + +async fn enqueue_job(state: &AppState, policy: QueuePolicy, _method: Method, uri: Uri, headers: HeaderMap, body: Bytes) -> Result { + let _permit = state.body_permits.acquire().await.map_err(|_| ServiceError::internal("body semaphore closed"))?; let job_id = new_job_id(); let tenant_id = required_header(&headers, "x-tenant-id")?; - let policy = optional_header(&headers, "x-ratelimit-policy").unwrap_or_else(|| "queue".to_string()); let model = optional_header(&headers, "x-model").unwrap_or_else(|| "default".to_string()); let callback_url = optional_header(&headers, "x-callback-url").unwrap_or_default(); + validate_callback_url(state, policy, &callback_url)?; let original_method = optional_header(&headers, "x-original-method").unwrap_or_else(|| "POST".to_string()); let original_path = optional_header(&headers, "x-original-path").unwrap_or_else(|| uri.path().to_string()); let request_headers = headers_to_json(&headers)?; let created_at = now_ms(); + let body_ref = store_body(state, &job_id, body).await?; + let stream_key = stream_for_request(state, &headers); let stream_id: String = state .redis .xadd( - state.cfg.stream_key.as_str(), + stream_key.as_str(), false, None::<()>, "*", vec![ ("job_id", Value::String(job_id.clone().into())), ("tenant_id", Value::String(tenant_id.into())), - ("policy", Value::String(policy.into())), + ("policy", Value::String(policy.as_str().into())), ("model", Value::String(model.into())), ("method", Value::String(original_method.into())), ("path", Value::String(original_path.into())), ("headers", Value::String(request_headers.into())), - ("body", Value::Bytes(body)), + ("body", Value::String(body_ref.body_base64.into())), + ("ref", Value::String(body_ref.object_ref.into())), + ("size", Value::Integer(body_ref.size as i64)), + ("storage", Value::String(body_ref.storage.into())), ("callback_url", Value::String(callback_url.into())), ("created_at", Value::Integer(created_at as i64)), ], ) .await?; + trim_stream(state, &stream_key).await?; - Ok(EnqueueResponse { - job_id: job_id.clone(), - stream_id, - status: "queued", - poll_url: format!("/v1/jobs/{job_id}"), + state.metrics.enqueue_total.fetch_add(1, Ordering::Relaxed); + match policy { + QueuePolicy::Queue => { + state.metrics.enqueue_queue_total.fetch_add(1, Ordering::Relaxed); + } + QueuePolicy::Wait => { + state.metrics.enqueue_wait_total.fetch_add(1, Ordering::Relaxed); + } + } + + Ok(AcceptedJob { + response: EnqueueResponse { + job_id: job_id.clone(), + stream_id, + stream_key, + status: "queued", + poll_url: format!("/v1/jobs/{job_id}"), + }, + created_at_ms: created_at, }) } @@ -366,35 +522,40 @@ fn spawn_workers(state: AppState) { } async fn worker_once(state: &AppState, consumer: &str) -> Result<(), ServiceError> { - let reply: XReadResponse = state - .redis - .xreadgroup_map( - state.cfg.consumer_group.as_str(), - consumer, - Some(5), - Some(1000), - false, - vec![state.cfg.stream_key.as_str()], - vec![">"], - ) - .await?; + let streams = worker_streams(state); + for (idx, stream) in streams.iter().enumerate() { + let block = if idx + 1 == streams.len() { 1000 } else { 10 }; + let processed = read_worker_stream(state, consumer, stream, block).await?; + if processed > 0 { + return Ok(()); + } + } + Ok(()) +} + +async fn read_worker_stream(state: &AppState, consumer: &str, stream: &str, block_ms: u64) -> Result { + let reply: XReadResponse = + state.redis.xreadgroup_map(state.cfg.consumer_group.as_str(), consumer, Some(5), Some(block_ms), false, vec![stream], vec![">"]).await?; + let mut processed = 0; for (_stream, entries) in reply { for (entry_id, fields) in entries { - match process_job(state, entry_id.as_str(), &fields).await { + match process_job(state, stream, entry_id.as_str(), &fields).await { Ok(()) => { - let _: i64 = state.redis.xack(state.cfg.stream_key.as_str(), state.cfg.consumer_group.as_str(), vec![entry_id.clone()]).await?; + let _: i64 = state.redis.xack(stream, state.cfg.consumer_group.as_str(), vec![entry_id.clone()]).await?; + processed += 1; } Err(e) => { tracing::warn!(stream_id = %entry_id, error = %e.message, "job processing failed"); + state.metrics.worker_failed_total.fetch_add(1, Ordering::Relaxed); } } } } - Ok(()) + Ok(processed) } -async fn process_job(state: &AppState, _stream_id: &str, fields: &HashMap) -> Result<(), ServiceError> { +async fn process_job(state: &AppState, _stream: &str, _stream_id: &str, fields: &HashMap) -> Result<(), ServiceError> { let Some(base) = state.cfg.upstream_base_url.as_deref() else { return Err(ServiceError::internal("upstream base URL is not configured")); }; @@ -403,7 +564,7 @@ async fn process_job(state: &AppState, _stream_id: &str, fields: &HashMap = serde_json::from_str(&headers_json).unwrap_or_default(); let url = format!("{}{}", base.trim_end_matches('/'), path); @@ -448,24 +609,145 @@ async fn process_job(state: &AppState, _stream_id: &str, fields: &HashMap serde_json::Value { + serde_json::json!({ + "job_id": result.job_id, + "status": result.status, + "http_status": result.http_status, + "headers": result.headers, + "body_base64": result.body_base64, + "result": result.body_base64, + "completed_at_ms": result.completed_at_ms, + "error": result.error, + }) +} + +async fn post_callback(state: &AppState, callback_url: &str, job_id: &str, body: &serde_json::Value) -> Result<(), ServiceError> { + state.http.post(callback_url).header("x-gateway-job-id", job_id).json(body).send().await?.error_for_status()?; + Ok(()) +} + +async fn enqueue_callback_retry(state: &AppState, callback_url: &str, job_id: &str, body: &serde_json::Value) -> Result<(), ServiceError> { + let body = serde_json::to_string(body).map_err(|e| ServiceError::internal(format!("serialize callback retry: {e}")))?; + let _: String = state + .redis + .xadd( + state.cfg.callback_retry_stream.as_str(), + false, + None::<()>, + "*", + vec![ + ("job_id", Value::String(job_id.to_string().into())), + ("callback_url", Value::String(callback_url.to_string().into())), + ("body", Value::String(body.into())), + ("created_at", Value::Integer(now_ms() as i64)), + ], + ) + .await?; + trim_stream(state, &state.cfg.callback_retry_stream).await?; + state.metrics.callback_retry_total.fetch_add(1, Ordering::Relaxed); Ok(()) } -async fn ensure_consumer_group(state: &AppState) -> Result<(), ServiceError> { - let res: FredResult = state.redis.xgroup_create(state.cfg.stream_key.as_str(), state.cfg.consumer_group.as_str(), "$", true).await; +fn spawn_callback_retry_worker(state: AppState) { + tokio::spawn(async move { + loop { + if let Err(e) = callback_retry_once(&state).await { + tracing::warn!(error = %e.message, "callback retry loop failed"); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + }); +} + +async fn callback_retry_once(state: &AppState) -> Result<(), ServiceError> { + let reply: XReadResponse = state + .redis + .xreadgroup_map( + state.cfg.callback_retry_group.as_str(), + state.cfg.consumer_name.as_str(), + Some(5), + Some(1000), + false, + vec![state.cfg.callback_retry_stream.as_str()], + vec![">"], + ) + .await?; + + for (_stream, entries) in reply { + for (entry_id, fields) in entries { + let job_id = field_string(&fields, "job_id").unwrap_or_default(); + let callback_url = field_string(&fields, "callback_url").unwrap_or_default(); + let body = field_string(&fields, "body").unwrap_or_else(|| "{}".to_string()); + let parsed = serde_json::from_str::(&body).unwrap_or_else(|_| serde_json::json!({ "body": body })); + match post_callback(state, &callback_url, &job_id, &parsed).await { + Ok(()) => { + let _: i64 = state.redis.xack(state.cfg.callback_retry_stream.as_str(), state.cfg.callback_retry_group.as_str(), vec![entry_id]).await?; + state.metrics.callback_retry_success_total.fetch_add(1, Ordering::Relaxed); + } + Err(e) => { + tracing::warn!(job_id = %job_id, error = %e.message, "callback retry failed"); + } + } + } + } + Ok(()) +} + +fn spawn_reclaimer(state: AppState) { + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(state.cfg.reclaim_interval_secs.max(1))); + loop { + interval.tick().await; + if let Err(e) = reclaim_once(&state).await { + tracing::warn!(error = %e.message, "stream reclaim failed"); + } + } + }); +} + +async fn reclaim_once(state: &AppState) -> Result<(), ServiceError> { + let consumer = format!("{}-reclaimer", state.cfg.consumer_name); + let min_idle_ms = state.cfg.reclaim_min_idle_secs.saturating_mul(1000); + for stream in worker_streams(state) { + let (_cursor, entries): (String, Vec<(String, HashMap)>) = + state.redis.xautoclaim_values(stream.as_str(), state.cfg.consumer_group.as_str(), consumer.as_str(), min_idle_ms, "0-0", Some(10), false).await?; + for (entry_id, fields) in entries { + match process_job(state, stream.as_str(), entry_id.as_str(), &fields).await { + Ok(()) => { + let _: i64 = state.redis.xack(stream.as_str(), state.cfg.consumer_group.as_str(), vec![entry_id]).await?; + state.metrics.reclaimed_total.fetch_add(1, Ordering::Relaxed); + } + Err(e) => { + tracing::warn!(stream = %stream, entry_id = %entry_id, error = %e.message, "reclaimed job failed"); + } + } + } + } + Ok(()) +} + +async fn ensure_consumer_groups(state: &AppState) -> Result<(), ServiceError> { + for stream in worker_streams(state) { + ensure_consumer_group(state, &stream, &state.cfg.consumer_group).await?; + } + ensure_consumer_group(state, &state.cfg.callback_retry_stream, &state.cfg.callback_retry_group).await?; + Ok(()) +} + +async fn ensure_consumer_group(state: &AppState, stream: &str, group: &str) -> Result<(), ServiceError> { + let res: FredResult = state.redis.xgroup_create(stream, group, "$", true).await; match res { Ok(_) => Ok(()), Err(e) if e.to_string().contains("BUSYGROUP") => Ok(()), @@ -488,7 +770,7 @@ async fn load_result(state: &AppState, job_id: &str) -> Result Result { +fn result_to_response(result: StoredResult, created_at_ms: u64) -> Result { let status = StatusCode::from_u16(result.http_status).unwrap_or(StatusCode::OK); let body = base64::engine::general_purpose::STANDARD.decode(result.body_base64).map_err(|e| ServiceError::internal(format!("decode result body: {e}")))?; let mut resp = (status, body).into_response(); @@ -498,9 +780,126 @@ fn result_to_response(result: StoredResult) -> Result { } } resp.headers_mut().insert("x-job-id", header_value(&result.job_id)?); + resp.headers_mut().insert("x-queue-wait-ms", header_value(&now_ms().saturating_sub(created_at_ms).to_string())?); Ok(resp) } +async fn store_body(state: &AppState, job_id: &str, body: Bytes) -> Result { + if body.len() <= state.cfg.inline_threshold || state.cfg.object_store_endpoint.is_none() { + return Ok(BodyLocation { + body_base64: base64::engine::general_purpose::STANDARD.encode(&body), + object_ref: String::new(), + size: body.len(), + storage: "inline", + }); + } + + let object_ref = format!("{}/{}/body.bin", state.cfg.object_store_prefix.trim_matches('/'), sanitize_key(job_id)); + let url = object_url(state, &object_ref); + let mut req = state.http.put(url).body(body.clone()); + if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { + req = req.header(name, value); + } + req.send().await?.error_for_status()?; + state.metrics.object_offload_total.fetch_add(1, Ordering::Relaxed); + Ok(BodyLocation { + body_base64: String::new(), + object_ref, + size: body.len(), + storage: "object", + }) +} + +async fn load_body(state: &AppState, fields: &HashMap) -> Result, ServiceError> { + let storage = field_string(fields, "storage").unwrap_or_else(|| "inline".to_string()); + if storage == "object" { + let object_ref = field_string(fields, "ref").ok_or_else(|| ServiceError::bad_request("job body is missing object ref"))?; + let url = object_url(state, &object_ref); + let mut req = state.http.get(url); + if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { + req = req.header(name, value); + } + return Ok(req.send().await?.error_for_status()?.bytes().await?.to_vec()); + } + + if let Some(body_base64) = field_string(fields, "body") { + return base64::engine::general_purpose::STANDARD.decode(body_base64).map_err(|e| ServiceError::bad_request(format!("decode job body: {e}"))); + } + Ok(field_bytes(fields, "body").unwrap_or_default()) +} + +fn object_url(state: &AppState, object_ref: &str) -> String { + format!( + "{}/{}/{}", + state.cfg.object_store_endpoint.as_deref().unwrap_or_default().trim_end_matches('/'), + state.cfg.object_store_bucket.trim_matches('/'), + object_ref.trim_start_matches('/') + ) +} + +fn object_auth_header(raw: &Option) -> Result, ServiceError> { + let Some(raw) = raw.as_deref() else { + return Ok(None); + }; + let Some((name, value)) = raw.split_once(':') else { + return Err(ServiceError::bad_request("AI_OBJECT_STORE_AUTH_HEADER must be `Header-Name: value`")); + }; + if HeaderName::try_from(name.trim()).is_err() || HeaderValue::from_str(value.trim()).is_err() { + return Err(ServiceError::bad_request("invalid object auth header")); + } + Ok(Some((name.trim().to_string(), value.trim().to_string()))) +} + +async fn trim_stream(state: &AppState, stream: &str) -> Result<(), ServiceError> { + if state.cfg.stream_max_len > 0 { + let _: i64 = state.redis.xtrim(stream, ("MAXLEN", "~", state.cfg.stream_max_len as i64)).await?; + } + Ok(()) +} + +fn stream_for_request(state: &AppState, headers: &HeaderMap) -> String { + if !state.cfg.enable_priority_streams { + return state.cfg.stream_key.clone(); + } + match optional_header(headers, "x-queue-priority").as_deref() { + Some("high") => state.cfg.high_priority_stream_key.clone(), + Some("low") => state.cfg.low_priority_stream_key.clone(), + _ => state.cfg.stream_key.clone(), + } +} + +fn worker_streams(state: &AppState) -> Vec { + if state.cfg.enable_priority_streams { + vec![ + state.cfg.high_priority_stream_key.clone(), + state.cfg.stream_key.clone(), + state.cfg.low_priority_stream_key.clone(), + ] + } else { + vec![state.cfg.stream_key.clone()] + } +} + +fn validate_callback_url(state: &AppState, policy: QueuePolicy, callback_url: &str) -> Result<(), ServiceError> { + if policy == QueuePolicy::Queue && callback_url.is_empty() { + return Err(ServiceError::bad_request("missing required header `x-callback-url` for queue policy")); + } + if !callback_url.is_empty() && state.cfg.require_https_callback && !callback_url.starts_with("https://") { + return Err(ServiceError::bad_request("x-callback-url must use https")); + } + Ok(()) +} + +async fn tenant_rate_limit(state: &AppState, tenant: &str) -> Result<(u64, u64), ServiceError> { + let key = format!("{}{}", state.cfg.tenant_rate_limit_prefix, sanitize_key(tenant)); + let rps: Option = state.redis.get(format!("{key}:rps")).await.unwrap_or(None); + let burst: Option = state.redis.get(format!("{key}:burst")).await.unwrap_or(None); + Ok(( + rps.and_then(|v| v.parse().ok()).unwrap_or(state.cfg.rate_limit_rps), + burst.and_then(|v| v.parse().ok()).unwrap_or(state.cfg.rate_limit_burst), + )) +} + fn required_header(headers: &HeaderMap, name: &str) -> Result { optional_header(headers, name).ok_or_else(|| ServiceError::bad_request(format!("missing required header `{name}`"))) } diff --git a/plugins/wasm/ai-gateway-queue/README.md b/plugins/wasm/ai-gateway-queue/README.md index 49acb33c..a72909d5 100644 --- a/plugins/wasm/ai-gateway-queue/README.md +++ b/plugins/wasm/ai-gateway-queue/README.md @@ -65,10 +65,20 @@ AI_RATE_LIMIT_BURST=200 AI_WAIT_TIMEOUT_SECS=60 AI_WORKER_CONCURRENCY=4 AI_MAX_BODY_BYTES=33554432 +AI_INLINE_THRESHOLD=131072 +AI_QUEUE_MAX_LEN=100000 +AI_RECLAIM_INTERVAL_SECS=30 +AI_RECLAIM_MIN_IDLE_SECS=30 ``` 如果不设置 `AI_UPSTREAM_BASE_URL`，队列任务仍会写入 Redis，但不会由本地 worker 消费。 +本地调试如果使用 HTTP 回调地址，可以临时加上： + +```bash +AI_REQUIRE_HTTPS_CALLBACK=false +``` + ## SpaceGate 配置 可参考： @@ -122,9 +132,10 @@ AI_MAX_BODY_BYTES=33554432 - `X-RateLimit-Policy`：必填，取值为 `abandon`、`queue`、`wait` - `X-Tenant-Id`：必填 -- `X-Callback-URL`：`queue` 场景下建议提供 +- `X-Callback-URL`：`queue` 场景下必填，默认要求 HTTPS - `X-Request-Timeout`：`wait` 场景下可选，单位为秒 - `X-Model`：可选，透传给外部服务 +- `X-Queue-Priority`：可选，启用优先级队列后可传 `high` 或 `low` Header 名称大小写不敏感；`X-RateLimit-Policy` 的值请使用小写。 @@ -154,7 +165,7 @@ curl -i http://localhost:9080/your/api \ curl -i http://localhost:9080/your/api \ -H 'X-RateLimit-Policy: queue' \ -H 'X-Tenant-Id: demo' \ - -H 'X-Callback-URL: http://localhost:9001/callback' \ + -H 'X-Callback-URL: https://example.com/callback' \ -d '{"prompt":"hello"}' ``` @@ -180,6 +191,17 @@ curl -i http://localhost:9080/your/api \ - `200`/`4xx`/`5xx`：`wait` 模式下，由外部服务返回 - `502`：外部服务不可达或调用失败 +`wait` 成功响应会带 `X-Job-Id` 和 `X-Queue-Wait-Ms`；`queue` 响应会带 `X-Job-Id` 和 `Location`。 + +## 生产化能力 + +- Redis Stream 支持 `MAXLEN ~` 裁剪，通过 `AI_QUEUE_MAX_LEN` 控制 +- Worker 崩溃后通过 `XAUTOCLAIM` 重认领 pending job +- 回调失败会进入 `AI_CALLBACK_RETRY_STREAM` 并由 retry worker 重试 +- 大 body 可通过 `AI_OBJECT_STORE_ENDPOINT` 卸载到对象存储，Redis Stream 中只保留 `ref` +- 可通过 Redis key 覆盖租户限流：`ai:tenant:ratelimit:{tenant}:rps` 和 `ai:tenant:ratelimit:{tenant}:burst` +- `/metrics` 暴露 Prometheus 文本指标 + ## 调试建议 - 先确认 `ai-gateway-service` 已启动并能连上 Redis diff --git a/plugins/wasm/ai-gateway-queue/src/lib.rs b/plugins/wasm/ai-gateway-queue/src/lib.rs index 4b9fd369..fff8ca6b 100644 --- a/plugins/wasm/ai-gateway-queue/src/lib.rs +++ b/plugins/wasm/ai-gateway-queue/src/lib.rs @@ -53,8 +53,9 @@ impl RootContext for AiGatewayRoot { let mut cfg = AiGatewayConfig::default(); for line in text.lines() { let Some((key, value)) = line.split_once(':') else { continue }; - let value = value.trim().trim_matches(['"', '\''].as_ref()); - match key.trim() { + let key = key.trim().trim_matches(['"', '\'', '{', ',', ' '].as_ref()); + let value = value.trim().trim_matches(['"', '\'', ',', ' '].as_ref()); + match key { "service_cluster" => cfg.service_cluster = value.to_string(), "service_authority" => cfg.service_authority = value.to_string(), "rate_limit_path" => cfg.rate_limit_path = value.to_string(), @@ -263,7 +264,7 @@ impl AiGatewayHttp { fn response_headers_for_client(&self) -> Vec<(String, String)> { let mut out = Vec::new(); - for name in ["content-type", "x-job-id", "retry-after"] { + for name in ["content-type", "x-job-id", "x-queue-wait-ms", "x-gateway-job-id", "retry-after", "location"] { if let Some(value) = self.get_http_call_response_header(name) { out.push((name.to_string(), value)); } From 61265a7b008aa95a4cbb2123ef6b1225521a3cc8 Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Thu, 21 May 2026 09:08:55 +0800 Subject: [PATCH 09/19] feat: use multipart object offload --- binary/ai-gateway-service/README.md | 9 ++ binary/ai-gateway-service/src/main.rs | 169 +++++++++++++++++++++++- plugins/wasm/ai-gateway-queue/README.md | 2 +- 3 files changed, 173 insertions(+), 7 deletions(-) diff --git a/binary/ai-gateway-service/README.md b/binary/ai-gateway-service/README.md index fa5221cc..d43eff29 100644 --- a/binary/ai-gateway-service/README.md +++ b/binary/ai-gateway-service/README.md @@ -54,9 +54,18 @@ Optional object offload variables: AI_OBJECT_STORE_ENDPOINT=http://127.0.0.1:9000 AI_OBJECT_STORE_BUCKET=ai-gateway-body AI_OBJECT_STORE_PREFIX=bodies +AI_OBJECT_MULTIPART_PART_SIZE=5242880 AI_OBJECT_STORE_AUTH_HEADER='Authorization: Bearer token' ``` +When `AI_OBJECT_STORE_ENDPOINT` is set and the body is larger than `AI_INLINE_THRESHOLD`, the service uses the S3-compatible multipart flow: + +```text +CreateMultipartUpload -> UploadPart* -> CompleteMultipartUpload +``` + +If any part upload or completion fails, the service sends `AbortMultipartUpload` before returning the enqueue error. The current implementation expects a MinIO/S3-compatible endpoint that accepts either unsigned requests or the configured static auth header. + Priority queues are disabled by default. Enable them and send `X-Queue-Priority: high|low` to route jobs to separate streams: ```bash diff --git a/binary/ai-gateway-service/src/main.rs b/binary/ai-gateway-service/src/main.rs index 082034c8..a9efe943 100644 --- a/binary/ai-gateway-service/src/main.rs +++ b/binary/ai-gateway-service/src/main.rs @@ -117,6 +117,8 @@ struct Args { object_store_bucket: String, #[arg(long, env = "AI_OBJECT_STORE_PREFIX", default_value = "bodies")] object_store_prefix: String, + #[arg(long, env = "AI_OBJECT_MULTIPART_PART_SIZE", default_value_t = 5 * 1024 * 1024)] + object_multipart_part_size: usize, #[arg(long, env = "AI_OBJECT_STORE_AUTH_HEADER")] object_store_auth_header: Option, } @@ -144,6 +146,7 @@ struct Metrics { worker_failed_total: AtomicU64, reclaimed_total: AtomicU64, object_offload_total: AtomicU64, + object_multipart_abort_total: AtomicU64, } #[derive(Debug, Serialize)] @@ -202,6 +205,12 @@ struct BodyLocation { storage: &'static str, } +#[derive(Debug)] +struct CompletedPart { + part_number: usize, + etag: String, +} + #[derive(Debug)] struct ServiceError { status: StatusCode, @@ -420,6 +429,7 @@ worker_completed_total {}\n\ worker_failed_total {}\n\ reclaimed_total {}\n\ object_offload_total {}\n\ +object_multipart_abort_total {}\n\ queue_depth {}\n\ queue_depth{{priority=\"high\"}} {}\n\ queue_depth{{priority=\"low\"}} {}\n\ @@ -436,6 +446,7 @@ callback_retry_depth {}\n", state.metrics.worker_failed_total.load(Ordering::Relaxed), state.metrics.reclaimed_total.load(Ordering::Relaxed), state.metrics.object_offload_total.load(Ordering::Relaxed), + state.metrics.object_multipart_abort_total.load(Ordering::Relaxed), queue_depth, high_queue_depth, low_queue_depth, @@ -795,12 +806,7 @@ async fn store_body(state: &AppState, job_id: &str, body: Bytes) -> Result) -> Result< Ok(field_bytes(fields, "body").unwrap_or_default()) } +async fn multipart_upload_body(state: &AppState, object_ref: &str, body: Bytes) -> Result<(), ServiceError> { + let upload_id = initiate_multipart_upload(state, object_ref).await?; + let upload_result = async { + let part_size = state.cfg.object_multipart_part_size.max(5 * 1024 * 1024); + let mut parts = Vec::new(); + for (idx, chunk) in body.chunks(part_size).enumerate() { + parts.push(upload_multipart_part(state, object_ref, &upload_id, idx + 1, chunk.to_vec()).await?); + } + complete_multipart_upload(state, object_ref, &upload_id, &parts).await + } + .await; + + if let Err(err) = upload_result { + if let Err(abort_err) = abort_multipart_upload(state, object_ref, &upload_id).await { + state.metrics.object_multipart_abort_total.fetch_add(1, Ordering::Relaxed); + tracing::warn!(object_ref = %object_ref, upload_id = %upload_id, error = %abort_err.message, "multipart upload abort failed"); + } else { + state.metrics.object_multipart_abort_total.fetch_add(1, Ordering::Relaxed); + } + return Err(err); + } + Ok(()) +} + +async fn initiate_multipart_upload(state: &AppState, object_ref: &str) -> Result { + let url = object_url_with_query(state, object_ref, "uploads"); + let mut req = state.http.post(url); + if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { + req = req.header(name, value); + } + let body = req.send().await?.error_for_status()?.text().await?; + extract_xml_tag(&body, "UploadId").ok_or_else(|| ServiceError::internal("multipart initiate response missing UploadId")) +} + +async fn upload_multipart_part(state: &AppState, object_ref: &str, upload_id: &str, part_number: usize, body: Vec) -> Result { + let query = format!("partNumber={part_number}&uploadId={}", encode_query_component(upload_id)); + let url = object_url_with_query(state, object_ref, &query); + let mut req = state.http.put(url).body(body); + if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { + req = req.header(name, value); + } + let resp = req.send().await?.error_for_status()?; + let etag = resp + .headers() + .get("etag") + .and_then(|value| value.to_str().ok()) + .map(ToOwned::to_owned) + .ok_or_else(|| ServiceError::internal("multipart upload part response missing ETag"))?; + Ok(CompletedPart { part_number, etag }) +} + +async fn complete_multipart_upload(state: &AppState, object_ref: &str, upload_id: &str, parts: &[CompletedPart]) -> Result<(), ServiceError> { + let query = format!("uploadId={}", encode_query_component(upload_id)); + let url = object_url_with_query(state, object_ref, &query); + let body = complete_multipart_xml(parts); + let mut req = state.http.post(url).header("content-type", "application/xml").body(body); + if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { + req = req.header(name, value); + } + req.send().await?.error_for_status()?; + Ok(()) +} + +async fn abort_multipart_upload(state: &AppState, object_ref: &str, upload_id: &str) -> Result<(), ServiceError> { + let query = format!("uploadId={}", encode_query_component(upload_id)); + let url = object_url_with_query(state, object_ref, &query); + let mut req = state.http.delete(url); + if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { + req = req.header(name, value); + } + req.send().await?.error_for_status()?; + Ok(()) +} + +fn complete_multipart_xml(parts: &[CompletedPart]) -> String { + let mut out = String::from(""); + for part in parts { + out.push_str(""); + out.push_str(""); + out.push_str(&part.part_number.to_string()); + out.push_str(""); + out.push_str(""); + out.push_str(&xml_escape(&part.etag)); + out.push_str(""); + out.push_str(""); + } + out.push_str(""); + out +} + fn object_url(state: &AppState, object_ref: &str) -> String { format!( "{}/{}/{}", @@ -837,6 +933,10 @@ fn object_url(state: &AppState, object_ref: &str) -> String { ) } +fn object_url_with_query(state: &AppState, object_ref: &str, query: &str) -> String { + format!("{}?{}", object_url(state, object_ref), query) +} + fn object_auth_header(raw: &Option) -> Result, ServiceError> { let Some(raw) = raw.as_deref() else { return Ok(None); @@ -850,6 +950,30 @@ fn object_auth_header(raw: &Option) -> Result, Ok(Some((name.trim().to_string(), value.trim().to_string()))) } +fn extract_xml_tag(xml: &str, tag: &str) -> Option { + let start_tag = format!("<{tag}>"); + let end_tag = format!(""); + let start = xml.find(&start_tag)? + start_tag.len(); + let end = xml[start..].find(&end_tag)? + start; + Some(xml[start..end].trim().to_string()) +} + +fn encode_query_component(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + for byte in input.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + out.push(byte as char); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + +fn xml_escape(input: &str) -> String { + input.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """).replace('\'', "'") +} + async fn trim_stream(state: &AppState, stream: &str) -> Result<(), ServiceError> { if state.cfg.stream_max_len > 0 { let _: i64 = state.redis.xtrim(stream, ("MAXLEN", "~", state.cfg.stream_max_len as i64)).await?; @@ -978,3 +1102,36 @@ fn field_bytes(fields: &HashMap, key: &str) -> Option> { _ => None, }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extracts_upload_id_from_multipart_xml() { + let xml = "a+b/c="; + assert_eq!(extract_xml_tag(xml, "UploadId").as_deref(), Some("a+b/c=")); + } + + #[test] + fn encodes_upload_id_for_query_string() { + assert_eq!(encode_query_component("a+b/c="), "a%2Bb%2Fc%3D"); + } + + #[test] + fn builds_complete_multipart_xml_with_escaped_etags() { + let parts = vec![ + CompletedPart { + part_number: 1, + etag: "\"abc&1\"".to_string(), + }, + CompletedPart { + part_number: 2, + etag: "\"def\"".to_string(), + }, + ]; + let xml = complete_multipart_xml(&parts); + assert!(xml.contains("1"abc&1"")); + assert!(xml.contains("2"def"")); + } +} diff --git a/plugins/wasm/ai-gateway-queue/README.md b/plugins/wasm/ai-gateway-queue/README.md index a72909d5..51eb4dae 100644 --- a/plugins/wasm/ai-gateway-queue/README.md +++ b/plugins/wasm/ai-gateway-queue/README.md @@ -198,7 +198,7 @@ curl -i http://localhost:9080/your/api \ - Redis Stream 支持 `MAXLEN ~` 裁剪，通过 `AI_QUEUE_MAX_LEN` 控制 - Worker 崩溃后通过 `XAUTOCLAIM` 重认领 pending job - 回调失败会进入 `AI_CALLBACK_RETRY_STREAM` 并由 retry worker 重试 -- 大 body 可通过 `AI_OBJECT_STORE_ENDPOINT` 卸载到对象存储，Redis Stream 中只保留 `ref` +- 大 body 可通过 `AI_OBJECT_STORE_ENDPOINT` 走 S3-compatible multipart 卸载，Redis Stream 中只保留 `ref` - 可通过 Redis key 覆盖租户限流：`ai:tenant:ratelimit:{tenant}:rps` 和 `ai:tenant:ratelimit:{tenant}:burst` - `/metrics` 暴露 Prometheus 文本指标 From e9cbac31fe60d5bce1a1368c5423204dc0c2f25b Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Thu, 21 May 2026 09:24:49 +0800 Subject: [PATCH 10/19] feat: stream queue request bodies --- binary/ai-gateway-service/README.md | 4 +- binary/ai-gateway-service/src/main.rs | 132 ++++++++++++++++-------- plugins/wasm/ai-gateway-queue/README.md | 1 + 3 files changed, 95 insertions(+), 42 deletions(-) diff --git a/binary/ai-gateway-service/README.md b/binary/ai-gateway-service/README.md index d43eff29..0a7419d9 100644 --- a/binary/ai-gateway-service/README.md +++ b/binary/ai-gateway-service/README.md @@ -13,7 +13,7 @@ It keeps Redis, worker execution, Pub/Sub waiting, callback delivery, and result - Returns `{ "allowed": bool, "retry_after_ms": number }`. - `POST /v1/queue/enqueue` - Requires `X-Callback-URL` by default. - - Stores selected headers and either inline base64 body or an object-store reference in Redis Stream. + - Streams the request body, then stores either inline base64 body or an object-store reference in Redis Stream. - Returns `202 Accepted` with `X-Job-Id`. - `POST /v1/queue/enqueue-and-wait` - Enqueues the job and waits for the worker result via Redis Pub/Sub. @@ -58,6 +58,8 @@ AI_OBJECT_MULTIPART_PART_SIZE=5242880 AI_OBJECT_STORE_AUTH_HEADER='Authorization: Bearer token' ``` +Request body reading is streaming. The service accumulates only the inline buffer until `AI_INLINE_THRESHOLD`; after that it starts multipart upload and flushes parts as `AI_OBJECT_MULTIPART_PART_SIZE` chunks become available. `AI_MAX_BODY_BYTES` is enforced while reading the stream. + When `AI_OBJECT_STORE_ENDPOINT` is set and the body is larger than `AI_INLINE_THRESHOLD`, the service uses the S3-compatible multipart flow: ```text diff --git a/binary/ai-gateway-service/src/main.rs b/binary/ai-gateway-service/src/main.rs index a9efe943..bd7988dd 100644 --- a/binary/ai-gateway-service/src/main.rs +++ b/binary/ai-gateway-service/src/main.rs @@ -4,7 +4,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use axum::body::Bytes; +use axum::body::Body; use axum::extract::{DefaultBodyLimit, Path, State}; use axum::http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode, Uri}; use axum::response::{IntoResponse, Response}; @@ -15,6 +15,7 @@ use clap::Parser; use fred::clients::{Client as FredClient, SubscriberClient}; use fred::prelude::*; use fred::types::streams::XReadResponse; +use futures_util::StreamExt; use serde::{Deserialize, Serialize}; use tokio::sync::Semaphore; use tower_http::trace::TraceLayer; @@ -238,6 +239,13 @@ impl ServiceError { message: message.into(), } } + + fn payload_too_large(message: impl Into) -> Self { + Self { + status: StatusCode::PAYLOAD_TOO_LARGE, + message: message.into(), + } + } } impl IntoResponse for ServiceError { @@ -343,7 +351,7 @@ async fn check_rate_limit(State(state): State, headers: HeaderMap, uri })) } -async fn enqueue(State(state): State, method: Method, uri: Uri, headers: HeaderMap, body: Bytes) -> Result { +async fn enqueue(State(state): State, method: Method, uri: Uri, headers: HeaderMap, body: Body) -> Result { let accepted = enqueue_job(&state, QueuePolicy::Queue, method, uri, headers, body).await?; let mut resp = (StatusCode::ACCEPTED, Json(&accepted.response)).into_response(); resp.headers_mut().insert("x-job-id", header_value(&accepted.response.job_id)?); @@ -351,7 +359,7 @@ async fn enqueue(State(state): State, method: Method, uri: Uri, header Ok(resp) } -async fn enqueue_and_wait(State(state): State, method: Method, uri: Uri, headers: HeaderMap, body: Bytes) -> Result { +async fn enqueue_and_wait(State(state): State, method: Method, uri: Uri, headers: HeaderMap, body: Body) -> Result { let timeout_secs = optional_header(&headers, "x-request-timeout").and_then(|v| v.parse::().ok()).unwrap_or(state.cfg.wait_timeout_secs); let accepted = enqueue_job(&state, QueuePolicy::Wait, method, uri, headers, body).await?; let channel = result_channel(&state, &accepted.response.job_id); @@ -455,7 +463,7 @@ callback_retry_depth {}\n", Ok((StatusCode::OK, [("content-type", "text/plain; version=0.0.4")], body).into_response()) } -async fn enqueue_job(state: &AppState, policy: QueuePolicy, _method: Method, uri: Uri, headers: HeaderMap, body: Bytes) -> Result { +async fn enqueue_job(state: &AppState, policy: QueuePolicy, _method: Method, uri: Uri, headers: HeaderMap, body: Body) -> Result { let _permit = state.body_permits.acquire().await.map_err(|_| ServiceError::internal("body semaphore closed"))?; let job_id = new_job_id(); let tenant_id = required_header(&headers, "x-tenant-id")?; @@ -795,24 +803,80 @@ fn result_to_response(result: StoredResult, created_at_ms: u64) -> Result Result { - if body.len() <= state.cfg.inline_threshold || state.cfg.object_store_endpoint.is_none() { +async fn store_body(state: &AppState, job_id: &str, body: Body) -> Result { + let object_ref = format!("{}/{}/body.bin", state.cfg.object_store_prefix.trim_matches('/'), sanitize_key(job_id)); + let mut stream = body.into_data_stream(); + let mut pending = Vec::new(); + let mut total_size = 0usize; + let mut upload_id = None; + let mut parts = Vec::new(); + let part_size = state.cfg.object_multipart_part_size.max(5 * 1024 * 1024); + + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| ServiceError::bad_request(format!("read request body: {e}")))?; + total_size = total_size.checked_add(chunk.len()).ok_or_else(|| ServiceError::payload_too_large("request body is too large"))?; + if total_size > state.cfg.max_body_bytes { + abort_upload_if_needed(state, &object_ref, upload_id.as_deref()).await; + return Err(ServiceError::payload_too_large(format!("request body exceeds max size {}", state.cfg.max_body_bytes))); + } + + if upload_id.is_none() { + if state.cfg.object_store_endpoint.is_some() && pending.len() + chunk.len() > state.cfg.inline_threshold { + pending.extend_from_slice(&chunk); + match initiate_multipart_upload(state, &object_ref).await { + Ok(id) => upload_id = Some(id), + Err(e) => return Err(e), + } + } else { + pending.extend_from_slice(&chunk); + continue; + } + } else { + pending.extend_from_slice(&chunk); + } + + if let Some(upload_id) = upload_id.as_deref() { + while pending.len() >= part_size { + let part_body = pending.drain(..part_size).collect::>(); + match upload_multipart_part(state, &object_ref, upload_id, parts.len() + 1, part_body).await { + Ok(part) => parts.push(part), + Err(e) => { + abort_upload_if_needed(state, &object_ref, Some(upload_id)).await; + return Err(e); + } + } + } + } + } + + if let Some(upload_id) = upload_id.as_deref() { + if !pending.is_empty() || parts.is_empty() { + match upload_multipart_part(state, &object_ref, upload_id, parts.len() + 1, pending).await { + Ok(part) => parts.push(part), + Err(e) => { + abort_upload_if_needed(state, &object_ref, Some(upload_id)).await; + return Err(e); + } + } + } + if let Err(e) = complete_multipart_upload(state, &object_ref, upload_id, &parts).await { + abort_upload_if_needed(state, &object_ref, Some(upload_id)).await; + return Err(e); + } + state.metrics.object_offload_total.fetch_add(1, Ordering::Relaxed); return Ok(BodyLocation { - body_base64: base64::engine::general_purpose::STANDARD.encode(&body), - object_ref: String::new(), - size: body.len(), - storage: "inline", + body_base64: String::new(), + object_ref, + size: total_size, + storage: "object", }); } - let object_ref = format!("{}/{}/body.bin", state.cfg.object_store_prefix.trim_matches('/'), sanitize_key(job_id)); - multipart_upload_body(state, &object_ref, body.clone()).await?; - state.metrics.object_offload_total.fetch_add(1, Ordering::Relaxed); Ok(BodyLocation { - body_base64: String::new(), - object_ref, - size: body.len(), - storage: "object", + body_base64: base64::engine::general_purpose::STANDARD.encode(&pending), + object_ref: String::new(), + size: total_size, + storage: "inline", }) } @@ -834,30 +898,6 @@ async fn load_body(state: &AppState, fields: &HashMap) -> Result< Ok(field_bytes(fields, "body").unwrap_or_default()) } -async fn multipart_upload_body(state: &AppState, object_ref: &str, body: Bytes) -> Result<(), ServiceError> { - let upload_id = initiate_multipart_upload(state, object_ref).await?; - let upload_result = async { - let part_size = state.cfg.object_multipart_part_size.max(5 * 1024 * 1024); - let mut parts = Vec::new(); - for (idx, chunk) in body.chunks(part_size).enumerate() { - parts.push(upload_multipart_part(state, object_ref, &upload_id, idx + 1, chunk.to_vec()).await?); - } - complete_multipart_upload(state, object_ref, &upload_id, &parts).await - } - .await; - - if let Err(err) = upload_result { - if let Err(abort_err) = abort_multipart_upload(state, object_ref, &upload_id).await { - state.metrics.object_multipart_abort_total.fetch_add(1, Ordering::Relaxed); - tracing::warn!(object_ref = %object_ref, upload_id = %upload_id, error = %abort_err.message, "multipart upload abort failed"); - } else { - state.metrics.object_multipart_abort_total.fetch_add(1, Ordering::Relaxed); - } - return Err(err); - } - Ok(()) -} - async fn initiate_multipart_upload(state: &AppState, object_ref: &str) -> Result { let url = object_url_with_query(state, object_ref, "uploads"); let mut req = state.http.post(url); @@ -908,6 +948,16 @@ async fn abort_multipart_upload(state: &AppState, object_ref: &str, upload_id: & Ok(()) } +async fn abort_upload_if_needed(state: &AppState, object_ref: &str, upload_id: Option<&str>) { + let Some(upload_id) = upload_id else { + return; + }; + state.metrics.object_multipart_abort_total.fetch_add(1, Ordering::Relaxed); + if let Err(abort_err) = abort_multipart_upload(state, object_ref, upload_id).await { + tracing::warn!(object_ref = %object_ref, upload_id = %upload_id, error = %abort_err.message, "multipart upload abort failed"); + } +} + fn complete_multipart_xml(parts: &[CompletedPart]) -> String { let mut out = String::from(""); for part in parts { diff --git a/plugins/wasm/ai-gateway-queue/README.md b/plugins/wasm/ai-gateway-queue/README.md index 51eb4dae..fbd13549 100644 --- a/plugins/wasm/ai-gateway-queue/README.md +++ b/plugins/wasm/ai-gateway-queue/README.md @@ -199,6 +199,7 @@ curl -i http://localhost:9080/your/api \ - Worker 崩溃后通过 `XAUTOCLAIM` 重认领 pending job - 回调失败会进入 `AI_CALLBACK_RETRY_STREAM` 并由 retry worker 重试 - 大 body 可通过 `AI_OBJECT_STORE_ENDPOINT` 走 S3-compatible multipart 卸载，Redis Stream 中只保留 `ref` +- `ai-gateway-service` 会流式读取请求体；当前 Wasm 插件转发到外部服务时仍受 `dispatch_http_call` 限制，会在插件侧拿到完整 body 后再发出调用 - 可通过 Redis key 覆盖租户限流：`ai:tenant:ratelimit:{tenant}:rps` 和 `ai:tenant:ratelimit:{tenant}:burst` - `/metrics` 暴露 Prometheus 文本指标 From 8c0274512416c1786b3216b1ef318b69b107d718 Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Thu, 21 May 2026 13:37:03 +0800 Subject: [PATCH 11/19] feat: enrich ai gateway queue metrics --- binary/ai-gateway-service/README.md | 20 +- binary/ai-gateway-service/src/main.rs | 499 ++++++++++++++++++++++-- plugins/wasm/ai-gateway-queue/README.md | 6 +- 3 files changed, 498 insertions(+), 27 deletions(-) diff --git a/binary/ai-gateway-service/README.md b/binary/ai-gateway-service/README.md index 0a7419d9..812505bc 100644 --- a/binary/ai-gateway-service/README.md +++ b/binary/ai-gateway-service/README.md @@ -21,7 +21,7 @@ It keeps Redis, worker execution, Pub/Sub waiting, callback delivery, and result - `GET /v1/jobs/{job_id}` - Returns the stored result JSON while the result key TTL is alive. - `GET /metrics` - - Returns Prometheus text metrics for queue depth, limits, callbacks, retries, and worker counters. + - Returns Prometheus text metrics for queue depth, PEL size, DLQ depth, enqueue latency, body size, waits, limits, callbacks, retries, object offload, and worker counters. ## Run @@ -45,7 +45,13 @@ AI_INLINE_THRESHOLD=131072 AI_QUEUE_MAX_LEN=100000 AI_RECLAIM_INTERVAL_SECS=30 AI_RECLAIM_MIN_IDLE_SECS=30 +AI_JOB_PROCESS_LEASE_SECS=120 +AI_JOB_MAX_DELIVERY_ATTEMPTS=5 AI_REQUIRE_HTTPS_CALLBACK=true +AI_CALLBACK_MAX_RETRY_ATTEMPTS=5 +AI_CALLBACK_RETRY_INITIAL_DELAY_MS=1000 +AI_CALLBACK_RETRY_MAX_DELAY_MS=60000 +AI_CALLBACK_RETRY_RECLAIM_IDLE_SECS=60 ``` Optional object offload variables: @@ -76,4 +82,14 @@ AI_QUEUE_HIGH_STREAM=ai:jobs:high AI_QUEUE_LOW_STREAM=ai:jobs:low ``` -Callback failures are written to `AI_CALLBACK_RETRY_STREAM` and retried by a local retry worker. Pending Redis Stream jobs are reclaimed with `XAUTOCLAIM` according to the reclaim settings. +Callback failures are written to `AI_CALLBACK_RETRY_STREAM` with `attempt`, `next_attempt_at_ms`, and `last_error`. The retry worker uses exponential backoff capped by `AI_CALLBACK_RETRY_MAX_DELAY_MS`, ACKs each retry record after handling it, and moves exhausted callbacks to `AI_CALLBACK_DLQ_STREAM`. Pending Redis Stream jobs are reclaimed with `XAUTOCLAIM` according to the reclaim settings. + +For job processing, each entry acquires a Redis lease key before upstream execution. Reclaimed entries that are already leased are skipped instead of being reprocessed, and jobs exceeding `AI_JOB_MAX_DELIVERY_ATTEMPTS` are moved to `AI_JOB_DLQ_STREAM`. + +`/metrics` includes the core signals needed to operate the queue: + +- `queue_depth`, `queue_depth{priority="high|low"}` for stream backlog. +- `pel_size`, `pel_size{priority="high|low"}`, and `callback_retry_pel_size` for unacked pending entries. +- `job_dlq_depth` and `callback_dlq_depth` for exhausted jobs and callbacks. +- `enqueue_latency_ms_*`, `enqueue_body_size_bytes_*`, `wait_total`, and `wait_timeout_total` for ingress and wait-mode health. +- `worker_processing_time_ms_*`, `worker_completed_total`, `worker_failed_total`, `reclaimed_total`, `lease_skip_total`, and `job_dlq_total` for worker health. diff --git a/binary/ai-gateway-service/src/main.rs b/binary/ai-gateway-service/src/main.rs index bd7988dd..cf3ad9bb 100644 --- a/binary/ai-gateway-service/src/main.rs +++ b/binary/ai-gateway-service/src/main.rs @@ -15,6 +15,7 @@ use clap::Parser; use fred::clients::{Client as FredClient, SubscriberClient}; use fred::prelude::*; use fred::types::streams::XReadResponse; +use fred::types::ExpireOptions; use futures_util::StreamExt; use serde::{Deserialize, Serialize}; use tokio::sync::Semaphore; @@ -78,10 +79,22 @@ struct Args { consumer_group: String, #[arg(long, env = "AI_QUEUE_CONSUMER", default_value = "ai-gateway-service")] consumer_name: String, + #[arg(long, env = "AI_JOB_DLQ_STREAM", default_value = "ai:job-dlq")] + job_dlq_stream: String, #[arg(long, env = "AI_CALLBACK_RETRY_STREAM", default_value = "ai:callback-retry")] callback_retry_stream: String, #[arg(long, env = "AI_CALLBACK_RETRY_GROUP", default_value = "ai-gateway-callbacks")] callback_retry_group: String, + #[arg(long, env = "AI_CALLBACK_DLQ_STREAM", default_value = "ai:callback-dlq")] + callback_dlq_stream: String, + #[arg(long, env = "AI_CALLBACK_MAX_RETRY_ATTEMPTS", default_value_t = 5)] + callback_max_retry_attempts: u32, + #[arg(long, env = "AI_CALLBACK_RETRY_INITIAL_DELAY_MS", default_value_t = 1000)] + callback_retry_initial_delay_ms: u64, + #[arg(long, env = "AI_CALLBACK_RETRY_MAX_DELAY_MS", default_value_t = 60_000)] + callback_retry_max_delay_ms: u64, + #[arg(long, env = "AI_CALLBACK_RETRY_RECLAIM_IDLE_SECS", default_value_t = 60)] + callback_retry_reclaim_idle_secs: u64, #[arg(long, env = "AI_RESULT_KEY_PREFIX", default_value = "result:")] result_key_prefix: String, #[arg(long, env = "AI_RESULT_CHANNEL_PREFIX", default_value = "result:")] @@ -110,6 +123,10 @@ struct Args { reclaim_interval_secs: u64, #[arg(long, env = "AI_RECLAIM_MIN_IDLE_SECS", default_value_t = 30)] reclaim_min_idle_secs: u64, + #[arg(long, env = "AI_JOB_PROCESS_LEASE_SECS", default_value_t = 120)] + job_process_lease_secs: u64, + #[arg(long, env = "AI_JOB_MAX_DELIVERY_ATTEMPTS", default_value_t = 5)] + job_max_delivery_attempts: u32, #[arg(long, env = "AI_REQUIRE_HTTPS_CALLBACK", default_value_t = true)] require_https_callback: bool, #[arg(long, env = "AI_OBJECT_STORE_ENDPOINT")] @@ -139,13 +156,35 @@ struct Metrics { enqueue_total: AtomicU64, enqueue_queue_total: AtomicU64, enqueue_wait_total: AtomicU64, + enqueue_latency_count: AtomicU64, + enqueue_latency_sum_ms: AtomicU64, + enqueue_latency_le_100_ms: AtomicU64, + enqueue_latency_le_500_ms: AtomicU64, + enqueue_latency_le_1000_ms: AtomicU64, + enqueue_latency_gt_1000_ms: AtomicU64, + body_size_le_10kb: AtomicU64, + body_size_le_128kb: AtomicU64, + body_size_le_5mb: AtomicU64, + body_size_gt_5mb: AtomicU64, + body_size_count: AtomicU64, + body_size_sum_bytes: AtomicU64, + wait_total: AtomicU64, wait_timeout_total: AtomicU64, callback_failure_total: AtomicU64, callback_retry_total: AtomicU64, callback_retry_success_total: AtomicU64, + callback_retry_dlq_total: AtomicU64, worker_completed_total: AtomicU64, worker_failed_total: AtomicU64, + worker_processing_count: AtomicU64, + worker_processing_sum_ms: AtomicU64, + worker_processing_le_1000_ms: AtomicU64, + worker_processing_le_5000_ms: AtomicU64, + worker_processing_le_30000_ms: AtomicU64, + worker_processing_gt_30000_ms: AtomicU64, reclaimed_total: AtomicU64, + job_dlq_total: AtomicU64, + lease_skip_total: AtomicU64, object_offload_total: AtomicU64, object_multipart_abort_total: AtomicU64, } @@ -361,6 +400,7 @@ async fn enqueue(State(state): State, method: Method, uri: Uri, header async fn enqueue_and_wait(State(state): State, method: Method, uri: Uri, headers: HeaderMap, body: Body) -> Result { let timeout_secs = optional_header(&headers, "x-request-timeout").and_then(|v| v.parse::().ok()).unwrap_or(state.cfg.wait_timeout_secs); + state.metrics.wait_total.fetch_add(1, Ordering::Relaxed); let accepted = enqueue_job(&state, QueuePolicy::Wait, method, uri, headers, body).await?; let channel = result_channel(&state, &accepted.response.job_id); let subscriber = build_subscriber_client(&state.cfg.redis_url)?; @@ -421,7 +461,13 @@ async fn metrics(State(state): State) -> Result Result { + let enqueue_started_at = now_ms(); let _permit = state.body_permits.acquire().await.map_err(|_| ServiceError::internal("body semaphore closed"))?; let job_id = new_job_id(); let tenant_id = required_header(&headers, "x-tenant-id")?; @@ -504,6 +611,8 @@ async fn enqueue_job(state: &AppState, policy: QueuePolicy, _method: Method, uri trim_stream(state, &stream_key).await?; state.metrics.enqueue_total.fetch_add(1, Ordering::Relaxed); + observe_enqueue_latency(&state.metrics, now_ms().saturating_sub(enqueue_started_at)); + observe_body_size(&state.metrics, body_ref.size); match policy { QueuePolicy::Queue => { state.metrics.enqueue_queue_total.fetch_add(1, Ordering::Relaxed); @@ -559,11 +668,11 @@ async fn read_worker_stream(state: &AppState, consumer: &str, stream: &str, bloc let mut processed = 0; for (_stream, entries) in reply { for (entry_id, fields) in entries { - match process_job(state, stream, entry_id.as_str(), &fields).await { - Ok(()) => { - let _: i64 = state.redis.xack(stream, state.cfg.consumer_group.as_str(), vec![entry_id.clone()]).await?; + match process_stream_entry(state, stream, entry_id.as_str(), &fields).await { + Ok(true) => { processed += 1; } + Ok(false) => {} Err(e) => { tracing::warn!(stream_id = %entry_id, error = %e.message, "job processing failed"); state.metrics.worker_failed_total.fetch_add(1, Ordering::Relaxed); @@ -574,6 +683,42 @@ async fn read_worker_stream(state: &AppState, consumer: &str, stream: &str, bloc Ok(processed) } +async fn process_stream_entry(state: &AppState, stream: &str, entry_id: &str, fields: &HashMap) -> Result { + let job_id = field_string(fields, "job_id").ok_or_else(|| ServiceError::bad_request("job missing job_id"))?; + let lease_owner = format!("{}:{stream}:{entry_id}:{}", state.cfg.consumer_name, now_ms()); + + if !acquire_job_lease(state, &job_id, &lease_owner).await? { + state.metrics.lease_skip_total.fetch_add(1, Ordering::Relaxed); + tracing::info!(job_id = %job_id, stream = %stream, entry_id = %entry_id, "job is already leased; skip reclaimed duplicate"); + return Ok(false); + } + + let attempt = increment_job_delivery_attempt(state, &job_id).await?; + if attempt > state.cfg.job_max_delivery_attempts { + enqueue_job_dlq(state, stream, entry_id, fields, attempt, "max_delivery_attempts_exceeded").await?; + ack_stream_entry(state, stream, entry_id).await?; + release_job_lease(state, &job_id).await; + state.metrics.job_dlq_total.fetch_add(1, Ordering::Relaxed); + return Ok(true); + } + + let processing_started_at = now_ms(); + match process_job(state, stream, entry_id, fields).await { + Ok(()) => { + observe_worker_processing(&state.metrics, now_ms().saturating_sub(processing_started_at)); + ack_stream_entry(state, stream, entry_id).await?; + clear_job_delivery_attempt(state, &job_id).await; + release_job_lease(state, &job_id).await; + Ok(true) + } + Err(e) => { + observe_worker_processing(&state.metrics, now_ms().saturating_sub(processing_started_at)); + release_job_lease(state, &job_id).await; + Err(e) + } + } +} + async fn process_job(state: &AppState, _stream: &str, _stream_id: &str, fields: &HashMap) -> Result<(), ServiceError> { let Some(base) = state.cfg.upstream_base_url.as_deref() else { return Err(ServiceError::internal("upstream base URL is not configured")); @@ -632,7 +777,7 @@ async fn process_job(state: &AppState, _stream: &str, _stream_id: &str, fields: if let Err(e) = post_callback(state, &callback_url, &job_id, &callback_body).await { tracing::warn!(job_id = %job_id, error = %e.message, "callback failed"); state.metrics.callback_failure_total.fetch_add(1, Ordering::Relaxed); - enqueue_callback_retry(state, &callback_url, &job_id, &callback_body).await?; + enqueue_callback_retry(state, &callback_url, &job_id, &callback_body, e.message.as_str()).await?; } } state.metrics.worker_completed_total.fetch_add(1, Ordering::Relaxed); @@ -657,8 +802,29 @@ async fn post_callback(state: &AppState, callback_url: &str, job_id: &str, body: Ok(()) } -async fn enqueue_callback_retry(state: &AppState, callback_url: &str, job_id: &str, body: &serde_json::Value) -> Result<(), ServiceError> { +async fn enqueue_callback_retry(state: &AppState, callback_url: &str, job_id: &str, body: &serde_json::Value, last_error: &str) -> Result<(), ServiceError> { let body = serde_json::to_string(body).map_err(|e| ServiceError::internal(format!("serialize callback retry: {e}")))?; + enqueue_callback_retry_raw( + state, + callback_url, + job_id, + &body, + 1, + now_ms().saturating_add(state.cfg.callback_retry_initial_delay_ms), + last_error, + ) + .await +} + +async fn enqueue_callback_retry_raw( + state: &AppState, + callback_url: &str, + job_id: &str, + body: &str, + attempt: u32, + next_attempt_at_ms: u64, + last_error: &str, +) -> Result<(), ServiceError> { let _: String = state .redis .xadd( @@ -669,7 +835,10 @@ async fn enqueue_callback_retry(state: &AppState, callback_url: &str, job_id: &s vec![ ("job_id", Value::String(job_id.to_string().into())), ("callback_url", Value::String(callback_url.to_string().into())), - ("body", Value::String(body.into())), + ("body", Value::String(body.to_string().into())), + ("attempt", Value::Integer(attempt as i64)), + ("next_attempt_at_ms", Value::Integer(next_attempt_at_ms as i64)), + ("last_error", Value::String(last_error.to_string().into())), ("created_at", Value::Integer(now_ms() as i64)), ], ) @@ -691,6 +860,7 @@ fn spawn_callback_retry_worker(state: AppState) { } async fn callback_retry_once(state: &AppState) -> Result<(), ServiceError> { + reclaim_callback_retries(state).await?; let reply: XReadResponse = state .redis .xreadgroup_map( @@ -706,24 +876,186 @@ async fn callback_retry_once(state: &AppState) -> Result<(), ServiceError> { for (_stream, entries) in reply { for (entry_id, fields) in entries { - let job_id = field_string(&fields, "job_id").unwrap_or_default(); - let callback_url = field_string(&fields, "callback_url").unwrap_or_default(); - let body = field_string(&fields, "body").unwrap_or_else(|| "{}".to_string()); - let parsed = serde_json::from_str::(&body).unwrap_or_else(|_| serde_json::json!({ "body": body })); - match post_callback(state, &callback_url, &job_id, &parsed).await { - Ok(()) => { - let _: i64 = state.redis.xack(state.cfg.callback_retry_stream.as_str(), state.cfg.callback_retry_group.as_str(), vec![entry_id]).await?; - state.metrics.callback_retry_success_total.fetch_add(1, Ordering::Relaxed); - } - Err(e) => { - tracing::warn!(job_id = %job_id, error = %e.message, "callback retry failed"); - } + process_callback_retry_entry(state, entry_id.as_str(), &fields).await?; + } + } + Ok(()) +} + +async fn reclaim_callback_retries(state: &AppState) -> Result<(), ServiceError> { + let consumer = format!("{}-callback-reclaimer", state.cfg.consumer_name); + let min_idle_ms = state.cfg.callback_retry_reclaim_idle_secs.saturating_mul(1000); + let (_cursor, entries): (String, Vec<(String, HashMap)>) = state + .redis + .xautoclaim_values( + state.cfg.callback_retry_stream.as_str(), + state.cfg.callback_retry_group.as_str(), + consumer.as_str(), + min_idle_ms, + "0-0", + Some(10), + false, + ) + .await?; + for (entry_id, fields) in entries { + process_callback_retry_entry(state, entry_id.as_str(), &fields).await?; + } + Ok(()) +} + +async fn process_callback_retry_entry(state: &AppState, entry_id: &str, fields: &HashMap) -> Result<(), ServiceError> { + let job_id = field_string(fields, "job_id").unwrap_or_default(); + let callback_url = field_string(fields, "callback_url").unwrap_or_default(); + let body = field_string(fields, "body").unwrap_or_else(|| "{}".to_string()); + let attempt = field_u32(fields, "attempt").unwrap_or(1); + let next_attempt_at_ms = field_u64(fields, "next_attempt_at_ms").unwrap_or(0); + let now = now_ms(); + + if next_attempt_at_ms > now { + let last_error = field_string(fields, "last_error").unwrap_or_default(); + enqueue_callback_retry_raw(state, &callback_url, &job_id, &body, attempt, next_attempt_at_ms, &last_error).await?; + ack_callback_retry(state, entry_id).await?; + return Ok(()); + } + + let parsed = serde_json::from_str::(&body).unwrap_or_else(|_| serde_json::json!({ "body": body })); + match post_callback(state, &callback_url, &job_id, &parsed).await { + Ok(()) => { + ack_callback_retry(state, entry_id).await?; + state.metrics.callback_retry_success_total.fetch_add(1, Ordering::Relaxed); + } + Err(e) => { + tracing::warn!(job_id = %job_id, attempt, error = %e.message, "callback retry failed"); + if attempt >= state.cfg.callback_max_retry_attempts { + enqueue_callback_dlq(state, &callback_url, &job_id, &parsed, attempt, &e.message).await?; + ack_callback_retry(state, entry_id).await?; + } else { + let next_attempt = attempt.saturating_add(1); + let delay_ms = callback_retry_delay_ms(state.cfg.callback_retry_initial_delay_ms, state.cfg.callback_retry_max_delay_ms, next_attempt); + let retry_body = serde_json::to_string(&parsed).unwrap_or_else(|_| "{}".to_string()); + enqueue_callback_retry_raw(state, &callback_url, &job_id, &retry_body, next_attempt, now.saturating_add(delay_ms), &e.message).await?; + ack_callback_retry(state, entry_id).await?; } } } Ok(()) } +async fn ack_callback_retry(state: &AppState, entry_id: &str) -> Result<(), ServiceError> { + let _: i64 = state.redis.xack(state.cfg.callback_retry_stream.as_str(), state.cfg.callback_retry_group.as_str(), vec![entry_id]).await?; + Ok(()) +} + +async fn enqueue_callback_dlq(state: &AppState, callback_url: &str, job_id: &str, body: &serde_json::Value, attempts: u32, final_error: &str) -> Result<(), ServiceError> { + let body = serde_json::to_string(body).map_err(|e| ServiceError::internal(format!("serialize callback dlq: {e}")))?; + let _: String = state + .redis + .xadd( + state.cfg.callback_dlq_stream.as_str(), + false, + None::<()>, + "*", + vec![ + ("job_id", Value::String(job_id.to_string().into())), + ("callback_url", Value::String(callback_url.to_string().into())), + ("body", Value::String(body.into())), + ("attempts", Value::Integer(attempts as i64)), + ("final_error", Value::String(final_error.to_string().into())), + ("failed_at", Value::Integer(now_ms() as i64)), + ], + ) + .await?; + trim_stream(state, &state.cfg.callback_dlq_stream).await?; + state.metrics.callback_retry_dlq_total.fetch_add(1, Ordering::Relaxed); + Ok(()) +} + +fn callback_retry_delay_ms(initial_delay_ms: u64, max_delay_ms: u64, attempt: u32) -> u64 { + let exponent = attempt.saturating_sub(1).min(16); + let multiplier = 1u64.checked_shl(exponent).unwrap_or(u64::MAX); + initial_delay_ms.saturating_mul(multiplier).min(max_delay_ms) +} + +async fn acquire_job_lease(state: &AppState, job_id: &str, owner: &str) -> Result { + let key = job_lease_key(job_id); + let result: Option = state + .redis + .set( + key, + owner, + Some(Expiration::EX(state.cfg.job_process_lease_secs.max(1) as i64)), + Some(SetOptions::NX), + false, + ) + .await?; + Ok(result.is_some()) +} + +async fn release_job_lease(state: &AppState, job_id: &str) { + let _: Result = state.redis.del(job_lease_key(job_id)).await; +} + +async fn increment_job_delivery_attempt(state: &AppState, job_id: &str) -> Result { + let key = job_attempt_key(job_id); + let attempt: i64 = state.redis.incr_by(key.as_str(), 1).await?; + let _: () = state.redis.expire(key.as_str(), state.cfg.result_ttl_secs.max(300) as i64, None::).await?; + Ok(attempt.max(0) as u32) +} + +async fn clear_job_delivery_attempt(state: &AppState, job_id: &str) { + let _: Result = state.redis.del(job_attempt_key(job_id)).await; +} + +async fn ack_stream_entry(state: &AppState, stream: &str, entry_id: &str) -> Result<(), ServiceError> { + let _: i64 = state.redis.xack(stream, state.cfg.consumer_group.as_str(), vec![entry_id]).await?; + Ok(()) +} + +async fn enqueue_job_dlq(state: &AppState, stream: &str, entry_id: &str, fields: &HashMap, attempts: u32, reason: &str) -> Result<(), ServiceError> { + let job_id = field_string(fields, "job_id").unwrap_or_default(); + let fields_json = stream_fields_to_json(fields)?; + let _: String = state + .redis + .xadd( + state.cfg.job_dlq_stream.as_str(), + false, + None::<()>, + "*", + vec![ + ("job_id", Value::String(job_id.into())), + ("source_stream", Value::String(stream.to_string().into())), + ("source_entry_id", Value::String(entry_id.to_string().into())), + ("attempts", Value::Integer(attempts as i64)), + ("reason", Value::String(reason.to_string().into())), + ("fields", Value::String(fields_json.into())), + ("failed_at", Value::Integer(now_ms() as i64)), + ], + ) + .await?; + trim_stream(state, &state.cfg.job_dlq_stream).await?; + Ok(()) +} + +fn stream_fields_to_json(fields: &HashMap) -> Result { + let mut out = HashMap::new(); + for (key, value) in fields { + if let Some(value) = field_string(fields, key) { + out.insert(key.clone(), value); + } else { + out.insert(key.clone(), format!("{value:?}")); + } + } + serde_json::to_string(&out).map_err(|e| ServiceError::internal(format!("serialize job dlq fields: {e}"))) +} + +fn job_lease_key(job_id: &str) -> String { + format!("ai:job:lease:{}", sanitize_key(job_id)) +} + +fn job_attempt_key(job_id: &str) -> String { + format!("ai:job:attempt:{}", sanitize_key(job_id)) +} + fn spawn_reclaimer(state: AppState) { tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(state.cfg.reclaim_interval_secs.max(1))); @@ -743,11 +1075,11 @@ async fn reclaim_once(state: &AppState) -> Result<(), ServiceError> { let (_cursor, entries): (String, Vec<(String, HashMap)>) = state.redis.xautoclaim_values(stream.as_str(), state.cfg.consumer_group.as_str(), consumer.as_str(), min_idle_ms, "0-0", Some(10), false).await?; for (entry_id, fields) in entries { - match process_job(state, stream.as_str(), entry_id.as_str(), &fields).await { - Ok(()) => { - let _: i64 = state.redis.xack(stream.as_str(), state.cfg.consumer_group.as_str(), vec![entry_id]).await?; + match process_stream_entry(state, stream.as_str(), entry_id.as_str(), &fields).await { + Ok(true) => { state.metrics.reclaimed_total.fetch_add(1, Ordering::Relaxed); } + Ok(false) => {} Err(e) => { tracing::warn!(stream = %stream, entry_id = %entry_id, error = %e.message, "reclaimed job failed"); } @@ -1031,6 +1363,84 @@ async fn trim_stream(state: &AppState, stream: &str) -> Result<(), ServiceError> Ok(()) } +async fn pending_size(state: &AppState, stream: &str) -> i64 { + pending_size_for_group(state, stream, state.cfg.consumer_group.as_str()).await +} + +async fn pending_size_for_group(state: &AppState, stream: &str, group: &str) -> i64 { + let raw: FredResult = state.redis.xpending(stream, group, ()).await; + match raw { + Ok(value) => pending_count_from_value(&value), + Err(e) => { + tracing::debug!(stream = %stream, group = %group, error = %e, "read stream pending size failed"); + 0 + } + } +} + +fn pending_count_from_value(value: &Value) -> i64 { + match value { + Value::Integer(value) => (*value).max(0), + Value::String(value) => value.parse::().unwrap_or(0).max(0), + Value::Bytes(value) => std::str::from_utf8(value).ok().and_then(|value| value.parse::().ok()).unwrap_or(0).max(0), + Value::Array(values) => values.first().map(pending_count_from_value).unwrap_or(0), + Value::Map(values) => values + .iter() + .find_map(|(key, value)| { + let key = key.as_str()?; + if key.eq_ignore_ascii_case("pending") || key.eq_ignore_ascii_case("count") { + Some(pending_count_from_value(value)) + } else { + None + } + }) + .unwrap_or(0), + _ => 0, + } +} + +fn observe_enqueue_latency(metrics: &Metrics, elapsed_ms: u64) { + metrics.enqueue_latency_count.fetch_add(1, Ordering::Relaxed); + metrics.enqueue_latency_sum_ms.fetch_add(elapsed_ms, Ordering::Relaxed); + if elapsed_ms <= 100 { + metrics.enqueue_latency_le_100_ms.fetch_add(1, Ordering::Relaxed); + } else if elapsed_ms <= 500 { + metrics.enqueue_latency_le_500_ms.fetch_add(1, Ordering::Relaxed); + } else if elapsed_ms <= 1000 { + metrics.enqueue_latency_le_1000_ms.fetch_add(1, Ordering::Relaxed); + } else { + metrics.enqueue_latency_gt_1000_ms.fetch_add(1, Ordering::Relaxed); + } +} + +fn observe_body_size(metrics: &Metrics, size: usize) { + metrics.body_size_count.fetch_add(1, Ordering::Relaxed); + metrics.body_size_sum_bytes.fetch_add(size as u64, Ordering::Relaxed); + if size <= 10 * 1024 { + metrics.body_size_le_10kb.fetch_add(1, Ordering::Relaxed); + } else if size <= 128 * 1024 { + metrics.body_size_le_128kb.fetch_add(1, Ordering::Relaxed); + } else if size <= 5 * 1024 * 1024 { + metrics.body_size_le_5mb.fetch_add(1, Ordering::Relaxed); + } else { + metrics.body_size_gt_5mb.fetch_add(1, Ordering::Relaxed); + } +} + +fn observe_worker_processing(metrics: &Metrics, elapsed_ms: u64) { + metrics.worker_processing_count.fetch_add(1, Ordering::Relaxed); + metrics.worker_processing_sum_ms.fetch_add(elapsed_ms, Ordering::Relaxed); + if elapsed_ms <= 1000 { + metrics.worker_processing_le_1000_ms.fetch_add(1, Ordering::Relaxed); + } else if elapsed_ms <= 5000 { + metrics.worker_processing_le_5000_ms.fetch_add(1, Ordering::Relaxed); + } else if elapsed_ms <= 30_000 { + metrics.worker_processing_le_30000_ms.fetch_add(1, Ordering::Relaxed); + } else { + metrics.worker_processing_gt_30000_ms.fetch_add(1, Ordering::Relaxed); + } +} + fn stream_for_request(state: &AppState, headers: &HeaderMap) -> String { if !state.cfg.enable_priority_streams { return state.cfg.stream_key.clone(); @@ -1153,6 +1563,19 @@ fn field_bytes(fields: &HashMap, key: &str) -> Option> { }) } +fn field_u64(fields: &HashMap, key: &str) -> Option { + fields.get(key).and_then(|value| match value { + Value::Integer(value) => (*value).try_into().ok(), + Value::String(value) => value.parse().ok(), + Value::Bytes(value) => std::str::from_utf8(value).ok().and_then(|value| value.parse().ok()), + _ => None, + }) +} + +fn field_u32(fields: &HashMap, key: &str) -> Option { + field_u64(fields, key).and_then(|value| value.try_into().ok()) +} + #[cfg(test)] mod tests { use super::*; @@ -1184,4 +1607,36 @@ mod tests { assert!(xml.contains("1"abc&1"")); assert!(xml.contains("2"def"")); } + + #[test] + fn callback_retry_delay_uses_exponential_backoff_with_cap() { + assert_eq!(callback_retry_delay_ms(1000, 60_000, 1), 1000); + assert_eq!(callback_retry_delay_ms(1000, 60_000, 3), 4000); + assert_eq!(callback_retry_delay_ms(1000, 5000, 8), 5000); + } + + #[test] + fn parses_xpending_summary_count() { + let value = Value::Array(vec![Value::Integer(7), Value::String("0-1".into()), Value::String("0-2".into())]); + assert_eq!(pending_count_from_value(&value), 7); + } + + #[test] + fn observes_histogram_buckets_as_non_overlapping_counts() { + let metrics = Metrics::default(); + observe_enqueue_latency(&metrics, 80); + observe_enqueue_latency(&metrics, 800); + observe_body_size(&metrics, 8 * 1024); + observe_body_size(&metrics, 256 * 1024); + observe_worker_processing(&metrics, 2000); + + assert_eq!(metrics.enqueue_latency_count.load(Ordering::Relaxed), 2); + assert_eq!(metrics.enqueue_latency_le_100_ms.load(Ordering::Relaxed), 1); + assert_eq!(metrics.enqueue_latency_le_1000_ms.load(Ordering::Relaxed), 1); + assert_eq!(metrics.body_size_count.load(Ordering::Relaxed), 2); + assert_eq!(metrics.body_size_le_10kb.load(Ordering::Relaxed), 1); + assert_eq!(metrics.body_size_le_5mb.load(Ordering::Relaxed), 1); + assert_eq!(metrics.worker_processing_count.load(Ordering::Relaxed), 1); + assert_eq!(metrics.worker_processing_le_5000_ms.load(Ordering::Relaxed), 1); + } } diff --git a/plugins/wasm/ai-gateway-queue/README.md b/plugins/wasm/ai-gateway-queue/README.md index fbd13549..d325e351 100644 --- a/plugins/wasm/ai-gateway-queue/README.md +++ b/plugins/wasm/ai-gateway-queue/README.md @@ -196,12 +196,12 @@ curl -i http://localhost:9080/your/api \ ## 生产化能力 - Redis Stream 支持 `MAXLEN ~` 裁剪，通过 `AI_QUEUE_MAX_LEN` 控制 -- Worker 崩溃后通过 `XAUTOCLAIM` 重认领 pending job -- 回调失败会进入 `AI_CALLBACK_RETRY_STREAM` 并由 retry worker 重试 +- Worker 崩溃后通过 `XAUTOCLAIM` 重认领 pending job，并通过 Redis 处理租约避免长任务被重复执行 +- 回调失败会进入 `AI_CALLBACK_RETRY_STREAM`，按指数退避重试，超过最大次数后进入 `AI_CALLBACK_DLQ_STREAM` - 大 body 可通过 `AI_OBJECT_STORE_ENDPOINT` 走 S3-compatible multipart 卸载，Redis Stream 中只保留 `ref` - `ai-gateway-service` 会流式读取请求体；当前 Wasm 插件转发到外部服务时仍受 `dispatch_http_call` 限制，会在插件侧拿到完整 body 后再发出调用 - 可通过 Redis key 覆盖租户限流：`ai:tenant:ratelimit:{tenant}:rps` 和 `ai:tenant:ratelimit:{tenant}:burst` -- `/metrics` 暴露 Prometheus 文本指标 +- `/metrics` 暴露 Prometheus 文本指标，包含队列深度、PEL、DLQ、入队延迟、body 大小、wait 超时、回调重试和 worker 处理耗时 ## 调试建议 From d3c18657d59f343b1794f3ceb376df16022603d4 Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Thu, 21 May 2026 14:04:48 +0800 Subject: [PATCH 12/19] feat: enhance ai queue configuration --- binary/ai-gateway-service/README.md | 41 +- binary/ai-gateway-service/src/app.rs | 34 + binary/ai-gateway-service/src/app/callback.rs | 191 ++ binary/ai-gateway-service/src/app/handlers.rs | 223 +++ binary/ai-gateway-service/src/app/metrics.rs | 85 + .../src/app/object_store.rs | 221 +++ binary/ai-gateway-service/src/app/queue.rs | 437 +++++ .../ai-gateway-service/src/app/ratelimit.rs | 52 + .../src/app/result_store.rs | 28 + binary/ai-gateway-service/src/app/runtime.rs | 40 + binary/ai-gateway-service/src/app/tests.rs | 85 + binary/ai-gateway-service/src/app/types.rs | 342 ++++ binary/ai-gateway-service/src/app/util.rs | 91 + binary/ai-gateway-service/src/main.rs | 1640 +---------------- plugins/wasm/Cargo.toml | 1 + plugins/wasm/ai-gateway-queue/Cargo.toml | 1 + plugins/wasm/ai-gateway-queue/README.md | 40 +- plugins/wasm/ai-gateway-queue/plugin.yaml | 7 + plugins/wasm/ai-gateway-queue/src/lib.rs | 312 +++- 19 files changed, 2204 insertions(+), 1667 deletions(-) create mode 100644 binary/ai-gateway-service/src/app.rs create mode 100644 binary/ai-gateway-service/src/app/callback.rs create mode 100644 binary/ai-gateway-service/src/app/handlers.rs create mode 100644 binary/ai-gateway-service/src/app/metrics.rs create mode 100644 binary/ai-gateway-service/src/app/object_store.rs create mode 100644 binary/ai-gateway-service/src/app/queue.rs create mode 100644 binary/ai-gateway-service/src/app/ratelimit.rs create mode 100644 binary/ai-gateway-service/src/app/result_store.rs create mode 100644 binary/ai-gateway-service/src/app/runtime.rs create mode 100644 binary/ai-gateway-service/src/app/tests.rs create mode 100644 binary/ai-gateway-service/src/app/types.rs create mode 100644 binary/ai-gateway-service/src/app/util.rs diff --git a/binary/ai-gateway-service/README.md b/binary/ai-gateway-service/README.md index 812505bc..f7e94c4d 100644 --- a/binary/ai-gateway-service/README.md +++ b/binary/ai-gateway-service/README.md @@ -38,11 +38,19 @@ REDIS_URL=redis://127.0.0.1/ AI_UPSTREAM_BASE_URL=http://127.0.0.1:9000 AI_RATE_LIMIT_RPS=100 AI_RATE_LIMIT_BURST=200 +AI_RATE_LIMIT_COST=1 AI_WAIT_TIMEOUT_SECS=60 AI_WORKER_CONCURRENCY=4 AI_MAX_BODY_BYTES=33554432 AI_INLINE_THRESHOLD=131072 AI_QUEUE_MAX_LEN=100000 +AI_ENABLE_PRIORITY_STREAMS=true +AI_QUEUE_DEFAULT_PRIORITY=normal +AI_QUEUE_HIGH_MODELS=gpt-4o,qwen-max +AI_QUEUE_LOW_TENANTS=free +AI_QUEUE_HIGH_WEIGHT=3 +AI_QUEUE_NORMAL_WEIGHT=1 +AI_QUEUE_LOW_WEIGHT=1 AI_RECLAIM_INTERVAL_SECS=30 AI_RECLAIM_MIN_IDLE_SECS=30 AI_JOB_PROCESS_LEASE_SECS=120 @@ -74,14 +82,45 @@ CreateMultipartUpload -> UploadPart* -> CompleteMultipartUpload If any part upload or completion fails, the service sends `AbortMultipartUpload` before returning the enqueue error. The current implementation expects a MinIO/S3-compatible endpoint that accepts either unsigned requests or the configured static auth header. -Priority queues are disabled by default. Enable them and send `X-Queue-Priority: high|low` to route jobs to separate streams: +Tenant rate-limit config can be overridden without restarting the service. The service checks Redis keys from most-specific to least-specific, using JSON or CSV values: + +```text +ai:tenant:ratelimit:{tenant}:model:{model}:path:{path}:policy:{policy} +ai:tenant:ratelimit:{tenant}:model:{model}:path:{path} +ai:tenant:ratelimit:{tenant}:model:{model}:policy:{policy} +ai:tenant:ratelimit:{tenant}:path:{path}:policy:{policy} +ai:tenant:ratelimit:{tenant}:model:{model} +ai:tenant:ratelimit:{tenant}:path:{path} +ai:tenant:ratelimit:{tenant}:policy:{policy} +ai:tenant:ratelimit:{tenant} +``` + +JSON value: + +```json +{"rps": 20, "burst": 40, "cost": 1} +``` + +CSV value: + +```text +20,40,1 +``` + +The old per-tenant keys are still supported as fallback: `ai:tenant:ratelimit:{tenant}:rps`, `:burst`, and `:cost`. + +Priority queues are disabled by default. Enable them and send `X-Queue-Priority: high|normal|low` to route jobs to separate streams, or configure model/tenant defaults: ```bash AI_ENABLE_PRIORITY_STREAMS=true AI_QUEUE_HIGH_STREAM=ai:jobs:high AI_QUEUE_LOW_STREAM=ai:jobs:low +AI_QUEUE_HIGH_MODELS=gpt-4o,qwen-max +AI_QUEUE_LOW_TENANTS=free ``` +Workers consume streams in weighted order. `AI_QUEUE_HIGH_WEIGHT`, `AI_QUEUE_NORMAL_WEIGHT`, and `AI_QUEUE_LOW_WEIGHT` control how often each priority stream is checked per loop. + Callback failures are written to `AI_CALLBACK_RETRY_STREAM` with `attempt`, `next_attempt_at_ms`, and `last_error`. The retry worker uses exponential backoff capped by `AI_CALLBACK_RETRY_MAX_DELAY_MS`, ACKs each retry record after handling it, and moves exhausted callbacks to `AI_CALLBACK_DLQ_STREAM`. Pending Redis Stream jobs are reclaimed with `XAUTOCLAIM` according to the reclaim settings. For job processing, each entry acquires a Redis lease key before upstream execution. Reclaimed entries that are already leased are skipped instead of being reprocessed, and jobs exceeding `AI_JOB_MAX_DELIVERY_ATTEMPTS` are moved to `AI_JOB_DLQ_STREAM`. diff --git a/binary/ai-gateway-service/src/app.rs b/binary/ai-gateway-service/src/app.rs new file mode 100644 index 00000000..6ca7a07f --- /dev/null +++ b/binary/ai-gateway-service/src/app.rs @@ -0,0 +1,34 @@ +use std::collections::HashMap; +use std::net::{IpAddr, SocketAddr}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use axum::body::Body; +use axum::extract::{DefaultBodyLimit, Path, State}; +use axum::http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode, Uri}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use base64::Engine; +use clap::Parser; +use fred::clients::{Client as FredClient, SubscriberClient}; +use fred::prelude::*; +use fred::types::streams::XReadResponse; +use fred::types::ExpireOptions; +use futures_util::StreamExt; +use serde::{Deserialize, Serialize}; +use tokio::sync::Semaphore; +use tower_http::trace::TraceLayer; + +include!("app/types.rs"); +include!("app/runtime.rs"); +include!("app/handlers.rs"); +include!("app/queue.rs"); +include!("app/callback.rs"); +include!("app/result_store.rs"); +include!("app/object_store.rs"); +include!("app/metrics.rs"); +include!("app/ratelimit.rs"); +include!("app/util.rs"); +include!("app/tests.rs"); diff --git a/binary/ai-gateway-service/src/app/callback.rs b/binary/ai-gateway-service/src/app/callback.rs new file mode 100644 index 00000000..e7ed0de2 --- /dev/null +++ b/binary/ai-gateway-service/src/app/callback.rs @@ -0,0 +1,191 @@ +fn callback_body(result: &StoredResult) -> serde_json::Value { + serde_json::json!({ + "job_id": result.job_id, + "status": result.status, + "http_status": result.http_status, + "headers": result.headers, + "body_base64": result.body_base64, + "result": result.body_base64, + "completed_at_ms": result.completed_at_ms, + "error": result.error, + }) +} + +async fn post_callback(state: &AppState, callback_url: &str, job_id: &str, body: &serde_json::Value) -> Result<(), ServiceError> { + state.http.post(callback_url).header("x-gateway-job-id", job_id).json(body).send().await?.error_for_status()?; + Ok(()) +} + +async fn enqueue_callback_retry(state: &AppState, callback_url: &str, job_id: &str, body: &serde_json::Value, last_error: &str) -> Result<(), ServiceError> { + let body = serde_json::to_string(body).map_err(|e| ServiceError::internal(format!("serialize callback retry: {e}")))?; + enqueue_callback_retry_raw( + state, + callback_url, + job_id, + &body, + 1, + now_ms().saturating_add(state.cfg.callback_retry_initial_delay_ms), + last_error, + ) + .await +} + +async fn enqueue_callback_retry_raw( + state: &AppState, + callback_url: &str, + job_id: &str, + body: &str, + attempt: u32, + next_attempt_at_ms: u64, + last_error: &str, +) -> Result<(), ServiceError> { + let _: String = state + .redis + .xadd( + state.cfg.callback_retry_stream.as_str(), + false, + None::<()>, + "*", + vec![ + ("job_id", Value::String(job_id.to_string().into())), + ("callback_url", Value::String(callback_url.to_string().into())), + ("body", Value::String(body.to_string().into())), + ("attempt", Value::Integer(attempt as i64)), + ("next_attempt_at_ms", Value::Integer(next_attempt_at_ms as i64)), + ("last_error", Value::String(last_error.to_string().into())), + ("created_at", Value::Integer(now_ms() as i64)), + ], + ) + .await?; + trim_stream(state, &state.cfg.callback_retry_stream).await?; + state.metrics.callback_retry_total.fetch_add(1, Ordering::Relaxed); + Ok(()) +} + +fn spawn_callback_retry_worker(state: AppState) { + tokio::spawn(async move { + loop { + if let Err(e) = callback_retry_once(&state).await { + tracing::warn!(error = %e.message, "callback retry loop failed"); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + }); +} + +async fn callback_retry_once(state: &AppState) -> Result<(), ServiceError> { + reclaim_callback_retries(state).await?; + let reply: XReadResponse = state + .redis + .xreadgroup_map( + state.cfg.callback_retry_group.as_str(), + state.cfg.consumer_name.as_str(), + Some(5), + Some(1000), + false, + vec![state.cfg.callback_retry_stream.as_str()], + vec![">"], + ) + .await?; + + for (_stream, entries) in reply { + for (entry_id, fields) in entries { + process_callback_retry_entry(state, entry_id.as_str(), &fields).await?; + } + } + Ok(()) +} + +async fn reclaim_callback_retries(state: &AppState) -> Result<(), ServiceError> { + let consumer = format!("{}-callback-reclaimer", state.cfg.consumer_name); + let min_idle_ms = state.cfg.callback_retry_reclaim_idle_secs.saturating_mul(1000); + let (_cursor, entries): (String, Vec<(String, HashMap)>) = state + .redis + .xautoclaim_values( + state.cfg.callback_retry_stream.as_str(), + state.cfg.callback_retry_group.as_str(), + consumer.as_str(), + min_idle_ms, + "0-0", + Some(10), + false, + ) + .await?; + for (entry_id, fields) in entries { + process_callback_retry_entry(state, entry_id.as_str(), &fields).await?; + } + Ok(()) +} + +async fn process_callback_retry_entry(state: &AppState, entry_id: &str, fields: &HashMap) -> Result<(), ServiceError> { + let job_id = field_string(fields, "job_id").unwrap_or_default(); + let callback_url = field_string(fields, "callback_url").unwrap_or_default(); + let body = field_string(fields, "body").unwrap_or_else(|| "{}".to_string()); + let attempt = field_u32(fields, "attempt").unwrap_or(1); + let next_attempt_at_ms = field_u64(fields, "next_attempt_at_ms").unwrap_or(0); + let now = now_ms(); + + if next_attempt_at_ms > now { + let last_error = field_string(fields, "last_error").unwrap_or_default(); + enqueue_callback_retry_raw(state, &callback_url, &job_id, &body, attempt, next_attempt_at_ms, &last_error).await?; + ack_callback_retry(state, entry_id).await?; + return Ok(()); + } + + let parsed = serde_json::from_str::(&body).unwrap_or_else(|_| serde_json::json!({ "body": body })); + match post_callback(state, &callback_url, &job_id, &parsed).await { + Ok(()) => { + ack_callback_retry(state, entry_id).await?; + state.metrics.callback_retry_success_total.fetch_add(1, Ordering::Relaxed); + } + Err(e) => { + tracing::warn!(job_id = %job_id, attempt, error = %e.message, "callback retry failed"); + if attempt >= state.cfg.callback_max_retry_attempts { + enqueue_callback_dlq(state, &callback_url, &job_id, &parsed, attempt, &e.message).await?; + ack_callback_retry(state, entry_id).await?; + } else { + let next_attempt = attempt.saturating_add(1); + let delay_ms = callback_retry_delay_ms(state.cfg.callback_retry_initial_delay_ms, state.cfg.callback_retry_max_delay_ms, next_attempt); + let retry_body = serde_json::to_string(&parsed).unwrap_or_else(|_| "{}".to_string()); + enqueue_callback_retry_raw(state, &callback_url, &job_id, &retry_body, next_attempt, now.saturating_add(delay_ms), &e.message).await?; + ack_callback_retry(state, entry_id).await?; + } + } + } + Ok(()) +} + +async fn ack_callback_retry(state: &AppState, entry_id: &str) -> Result<(), ServiceError> { + let _: i64 = state.redis.xack(state.cfg.callback_retry_stream.as_str(), state.cfg.callback_retry_group.as_str(), vec![entry_id]).await?; + Ok(()) +} + +async fn enqueue_callback_dlq(state: &AppState, callback_url: &str, job_id: &str, body: &serde_json::Value, attempts: u32, final_error: &str) -> Result<(), ServiceError> { + let body = serde_json::to_string(body).map_err(|e| ServiceError::internal(format!("serialize callback dlq: {e}")))?; + let _: String = state + .redis + .xadd( + state.cfg.callback_dlq_stream.as_str(), + false, + None::<()>, + "*", + vec![ + ("job_id", Value::String(job_id.to_string().into())), + ("callback_url", Value::String(callback_url.to_string().into())), + ("body", Value::String(body.into())), + ("attempts", Value::Integer(attempts as i64)), + ("final_error", Value::String(final_error.to_string().into())), + ("failed_at", Value::Integer(now_ms() as i64)), + ], + ) + .await?; + trim_stream(state, &state.cfg.callback_dlq_stream).await?; + state.metrics.callback_retry_dlq_total.fetch_add(1, Ordering::Relaxed); + Ok(()) +} + +fn callback_retry_delay_ms(initial_delay_ms: u64, max_delay_ms: u64, attempt: u32) -> u64 { + let exponent = attempt.saturating_sub(1).min(16); + let multiplier = 1u64.checked_shl(exponent).unwrap_or(u64::MAX); + initial_delay_ms.saturating_mul(multiplier).min(max_delay_ms) +} diff --git a/binary/ai-gateway-service/src/app/handlers.rs b/binary/ai-gateway-service/src/app/handlers.rs new file mode 100644 index 00000000..208bc797 --- /dev/null +++ b/binary/ai-gateway-service/src/app/handlers.rs @@ -0,0 +1,223 @@ +async fn healthz() -> &'static str { + "ok" +} + +async fn check_rate_limit(State(state): State, headers: HeaderMap, uri: Uri) -> Result, ServiceError> { + let tenant = required_header(&headers, "x-tenant-id")?; + let model = optional_header(&headers, "x-model").unwrap_or_else(|| "default".to_string()); + let path = optional_header(&headers, "x-original-path").unwrap_or_else(|| uri.path().to_string()); + let policy = optional_header(&headers, "x-ratelimit-policy").unwrap_or_else(|| "abandon".to_string()); + let rate_limit = tenant_rate_limit(&state, &tenant, &model, &path, &policy).await?; + let key = sanitize_key(&format!("{tenant}:{model}:{path}")); + let tokens_key = format!("ai:ratelimit:{key}:tokens"); + let ts_key = format!("ai:ratelimit:{key}:ts"); + let now = now_ms(); + + let out: Vec = state + .redis + .eval( + TOKEN_BUCKET_LUA, + vec![tokens_key, ts_key], + vec![rate_limit.rps.to_string(), rate_limit.burst.to_string(), now.to_string(), rate_limit.cost.to_string()], + ) + .await?; + + let allowed = out.first().copied().unwrap_or(0) == 1; + if !allowed { + state.metrics.rate_limited_total.fetch_add(1, Ordering::Relaxed); + } + Ok(Json(RateLimitResponse { + allowed, + remaining_tokens_milli: out.get(1).copied().unwrap_or(0), + retry_after_ms: out.get(2).copied().unwrap_or(0), + })) +} + +async fn enqueue(State(state): State, method: Method, uri: Uri, headers: HeaderMap, body: Body) -> Result { + let accepted = enqueue_job(&state, QueuePolicy::Queue, method, uri, headers, body).await?; + let mut resp = (StatusCode::ACCEPTED, Json(&accepted.response)).into_response(); + resp.headers_mut().insert("x-job-id", header_value(&accepted.response.job_id)?); + resp.headers_mut().insert("location", header_value(&accepted.response.poll_url)?); + Ok(resp) +} + +async fn enqueue_and_wait(State(state): State, method: Method, uri: Uri, headers: HeaderMap, body: Body) -> Result { + let timeout_secs = optional_header(&headers, "x-request-timeout").and_then(|v| v.parse::().ok()).unwrap_or(state.cfg.wait_timeout_secs); + state.metrics.wait_total.fetch_add(1, Ordering::Relaxed); + let accepted = enqueue_job(&state, QueuePolicy::Wait, method, uri, headers, body).await?; + let channel = result_channel(&state, &accepted.response.job_id); + let subscriber = build_subscriber_client(&state.cfg.redis_url)?; + let _subscriber_task = subscriber.init().await?; + subscriber.subscribe(channel.as_str()).await?; + + if let Some(result) = load_result(&state, &accepted.response.job_id).await? { + let _ = subscriber.quit().await; + return Ok(result_to_response(result, accepted.created_at_ms)?); + } + + let mut messages = subscriber.message_rx(); + let wait = tokio::time::timeout(Duration::from_secs(timeout_secs), async { + loop { + let message = messages.recv().await.map_err(|e| ServiceError::internal(format!("pubsub receive: {e}")))?; + if &*message.channel == channel.as_str() { + return Ok::<(), ServiceError>(()); + } + } + }) + .await; + match wait { + Ok(Ok(())) => { + let _ = subscriber.quit().await; + if let Some(result) = load_result(&state, &accepted.response.job_id).await? { + Ok(result_to_response(result, accepted.created_at_ms)?) + } else { + Err(ServiceError::gateway_timeout(format!( + "job {} completed notification received but result is missing", + accepted.response.job_id + ))) + } + } + _ => { + let _ = subscriber.quit().await; + state.metrics.wait_timeout_total.fetch_add(1, Ordering::Relaxed); + let waited_ms = now_ms().saturating_sub(accepted.created_at_ms); + let body = Json(serde_json::json!({ + "error": "timeout", + "job_id": accepted.response.job_id, + "poll_url": accepted.response.poll_url, + "waited_ms": waited_ms, + "message": "Job is still processing. Switch to queue mode with a callback for long tasks." + })); + Ok((StatusCode::GATEWAY_TIMEOUT, body).into_response()) + } + } +} + +async fn get_job(State(state): State, Path(job_id): Path) -> Result { + match load_result(&state, &job_id).await? { + Some(result) => Ok(Json(result).into_response()), + None => Ok((StatusCode::NOT_FOUND, Json(serde_json::json!({ "error": "not_found", "job_id": job_id }))).into_response()), + } +} + +async fn metrics(State(state): State) -> Result { + let queue_depth: i64 = state.redis.xlen(state.cfg.stream_key.as_str()).await.unwrap_or_default(); + let high_queue_depth: i64 = state.redis.xlen(state.cfg.high_priority_stream_key.as_str()).await.unwrap_or_default(); + let low_queue_depth: i64 = state.redis.xlen(state.cfg.low_priority_stream_key.as_str()).await.unwrap_or_default(); + let job_dlq_depth: i64 = state.redis.xlen(state.cfg.job_dlq_stream.as_str()).await.unwrap_or_default(); + let callback_retry_depth: i64 = state.redis.xlen(state.cfg.callback_retry_stream.as_str()).await.unwrap_or_default(); + let callback_dlq_depth: i64 = state.redis.xlen(state.cfg.callback_dlq_stream.as_str()).await.unwrap_or_default(); + let pel_size = pending_size(&state, &state.cfg.stream_key).await; + let high_pel_size = pending_size(&state, &state.cfg.high_priority_stream_key).await; + let low_pel_size = pending_size(&state, &state.cfg.low_priority_stream_key).await; + let callback_retry_pel_size = pending_size_for_group(&state, &state.cfg.callback_retry_stream, &state.cfg.callback_retry_group).await; + + let body = format!( + "\ +rate_limited_total {}\n\ +enqueue_total {}\n\ +enqueue_total{{policy=\"queue\"}} {}\n\ +enqueue_total{{policy=\"wait\"}} {}\n\ +enqueue_total{{priority=\"high\"}} {}\n\ +enqueue_total{{priority=\"normal\"}} {}\n\ +enqueue_total{{priority=\"low\"}} {}\n\ +enqueue_latency_ms_count {}\n\ +enqueue_latency_ms_sum {}\n\ +enqueue_latency_ms_bucket{{le=\"100\"}} {}\n\ +enqueue_latency_ms_bucket{{le=\"500\"}} {}\n\ +enqueue_latency_ms_bucket{{le=\"1000\"}} {}\n\ +enqueue_latency_ms_bucket{{le=\"+Inf\"}} {}\n\ +enqueue_body_size_bytes_count {}\n\ +enqueue_body_size_bytes_sum {}\n\ +enqueue_body_size_bytes_bucket{{le=\"10240\"}} {}\n\ +enqueue_body_size_bytes_bucket{{le=\"131072\"}} {}\n\ +enqueue_body_size_bytes_bucket{{le=\"5242880\"}} {}\n\ +enqueue_body_size_bytes_bucket{{le=\"+Inf\"}} {}\n\ +wait_total {}\n\ +wait_timeout_total {}\n\ +callback_failure_total {}\n\ +callback_retry_total {}\n\ +callback_retry_success_total {}\n\ +callback_retry_dlq_total {}\n\ +worker_completed_total {}\n\ +worker_failed_total {}\n\ +worker_processing_time_ms_count {}\n\ +worker_processing_time_ms_sum {}\n\ +worker_processing_time_ms_bucket{{le=\"1000\"}} {}\n\ +worker_processing_time_ms_bucket{{le=\"5000\"}} {}\n\ +worker_processing_time_ms_bucket{{le=\"30000\"}} {}\n\ +worker_processing_time_ms_bucket{{le=\"+Inf\"}} {}\n\ +reclaimed_total {}\n\ +job_dlq_total {}\n\ +lease_skip_total {}\n\ +object_offload_total {}\n\ +object_multipart_abort_total {}\n\ +queue_depth {}\n\ +queue_depth{{priority=\"normal\"}} {}\n\ +queue_depth{{priority=\"high\"}} {}\n\ +queue_depth{{priority=\"low\"}} {}\n\ +pel_size {}\n\ +pel_size{{priority=\"normal\"}} {}\n\ +pel_size{{priority=\"high\"}} {}\n\ +pel_size{{priority=\"low\"}} {}\n\ +job_dlq_depth {}\n\ +callback_retry_depth {}\n\ +callback_retry_pel_size {}\n\ +callback_dlq_depth {}\n", + state.metrics.rate_limited_total.load(Ordering::Relaxed), + state.metrics.enqueue_total.load(Ordering::Relaxed), + state.metrics.enqueue_queue_total.load(Ordering::Relaxed), + state.metrics.enqueue_wait_total.load(Ordering::Relaxed), + state.metrics.enqueue_priority_high_total.load(Ordering::Relaxed), + state.metrics.enqueue_priority_normal_total.load(Ordering::Relaxed), + state.metrics.enqueue_priority_low_total.load(Ordering::Relaxed), + state.metrics.enqueue_latency_count.load(Ordering::Relaxed), + state.metrics.enqueue_latency_sum_ms.load(Ordering::Relaxed), + state.metrics.enqueue_latency_le_100_ms.load(Ordering::Relaxed), + state.metrics.enqueue_latency_le_100_ms.load(Ordering::Relaxed) + state.metrics.enqueue_latency_le_500_ms.load(Ordering::Relaxed), + state.metrics.enqueue_latency_le_100_ms.load(Ordering::Relaxed) + + state.metrics.enqueue_latency_le_500_ms.load(Ordering::Relaxed) + + state.metrics.enqueue_latency_le_1000_ms.load(Ordering::Relaxed), + state.metrics.enqueue_latency_count.load(Ordering::Relaxed), + state.metrics.body_size_count.load(Ordering::Relaxed), + state.metrics.body_size_sum_bytes.load(Ordering::Relaxed), + state.metrics.body_size_le_10kb.load(Ordering::Relaxed), + state.metrics.body_size_le_10kb.load(Ordering::Relaxed) + state.metrics.body_size_le_128kb.load(Ordering::Relaxed), + state.metrics.body_size_le_10kb.load(Ordering::Relaxed) + state.metrics.body_size_le_128kb.load(Ordering::Relaxed) + state.metrics.body_size_le_5mb.load(Ordering::Relaxed), + state.metrics.body_size_count.load(Ordering::Relaxed), + state.metrics.wait_total.load(Ordering::Relaxed), + state.metrics.wait_timeout_total.load(Ordering::Relaxed), + state.metrics.callback_failure_total.load(Ordering::Relaxed), + state.metrics.callback_retry_total.load(Ordering::Relaxed), + state.metrics.callback_retry_success_total.load(Ordering::Relaxed), + state.metrics.callback_retry_dlq_total.load(Ordering::Relaxed), + state.metrics.worker_completed_total.load(Ordering::Relaxed), + state.metrics.worker_failed_total.load(Ordering::Relaxed), + state.metrics.worker_processing_count.load(Ordering::Relaxed), + state.metrics.worker_processing_sum_ms.load(Ordering::Relaxed), + state.metrics.worker_processing_le_1000_ms.load(Ordering::Relaxed), + state.metrics.worker_processing_le_1000_ms.load(Ordering::Relaxed) + state.metrics.worker_processing_le_5000_ms.load(Ordering::Relaxed), + state.metrics.worker_processing_le_1000_ms.load(Ordering::Relaxed) + + state.metrics.worker_processing_le_5000_ms.load(Ordering::Relaxed) + + state.metrics.worker_processing_le_30000_ms.load(Ordering::Relaxed), + state.metrics.worker_processing_count.load(Ordering::Relaxed), + state.metrics.reclaimed_total.load(Ordering::Relaxed), + state.metrics.job_dlq_total.load(Ordering::Relaxed), + state.metrics.lease_skip_total.load(Ordering::Relaxed), + state.metrics.object_offload_total.load(Ordering::Relaxed), + state.metrics.object_multipart_abort_total.load(Ordering::Relaxed), + queue_depth, + queue_depth, + high_queue_depth, + low_queue_depth, + pel_size, + pel_size, + high_pel_size, + low_pel_size, + job_dlq_depth, + callback_retry_depth, + callback_retry_pel_size, + callback_dlq_depth, + ); + Ok((StatusCode::OK, [("content-type", "text/plain; version=0.0.4")], body).into_response()) +} diff --git a/binary/ai-gateway-service/src/app/metrics.rs b/binary/ai-gateway-service/src/app/metrics.rs new file mode 100644 index 00000000..887866e1 --- /dev/null +++ b/binary/ai-gateway-service/src/app/metrics.rs @@ -0,0 +1,85 @@ +async fn trim_stream(state: &AppState, stream: &str) -> Result<(), ServiceError> { + if state.cfg.stream_max_len > 0 { + let _: i64 = state.redis.xtrim(stream, ("MAXLEN", "~", state.cfg.stream_max_len as i64)).await?; + } + Ok(()) +} + +async fn pending_size(state: &AppState, stream: &str) -> i64 { + pending_size_for_group(state, stream, state.cfg.consumer_group.as_str()).await +} + +async fn pending_size_for_group(state: &AppState, stream: &str, group: &str) -> i64 { + let raw: FredResult = state.redis.xpending(stream, group, ()).await; + match raw { + Ok(value) => pending_count_from_value(&value), + Err(e) => { + tracing::debug!(stream = %stream, group = %group, error = %e, "read stream pending size failed"); + 0 + } + } +} + +fn pending_count_from_value(value: &Value) -> i64 { + match value { + Value::Integer(value) => (*value).max(0), + Value::String(value) => value.parse::().unwrap_or(0).max(0), + Value::Bytes(value) => std::str::from_utf8(value).ok().and_then(|value| value.parse::().ok()).unwrap_or(0).max(0), + Value::Array(values) => values.first().map(pending_count_from_value).unwrap_or(0), + Value::Map(values) => values + .iter() + .find_map(|(key, value)| { + let key = key.as_str()?; + if key.eq_ignore_ascii_case("pending") || key.eq_ignore_ascii_case("count") { + Some(pending_count_from_value(value)) + } else { + None + } + }) + .unwrap_or(0), + _ => 0, + } +} + +fn observe_enqueue_latency(metrics: &Metrics, elapsed_ms: u64) { + metrics.enqueue_latency_count.fetch_add(1, Ordering::Relaxed); + metrics.enqueue_latency_sum_ms.fetch_add(elapsed_ms, Ordering::Relaxed); + if elapsed_ms <= 100 { + metrics.enqueue_latency_le_100_ms.fetch_add(1, Ordering::Relaxed); + } else if elapsed_ms <= 500 { + metrics.enqueue_latency_le_500_ms.fetch_add(1, Ordering::Relaxed); + } else if elapsed_ms <= 1000 { + metrics.enqueue_latency_le_1000_ms.fetch_add(1, Ordering::Relaxed); + } else { + metrics.enqueue_latency_gt_1000_ms.fetch_add(1, Ordering::Relaxed); + } +} + +fn observe_body_size(metrics: &Metrics, size: usize) { + metrics.body_size_count.fetch_add(1, Ordering::Relaxed); + metrics.body_size_sum_bytes.fetch_add(size as u64, Ordering::Relaxed); + if size <= 10 * 1024 { + metrics.body_size_le_10kb.fetch_add(1, Ordering::Relaxed); + } else if size <= 128 * 1024 { + metrics.body_size_le_128kb.fetch_add(1, Ordering::Relaxed); + } else if size <= 5 * 1024 * 1024 { + metrics.body_size_le_5mb.fetch_add(1, Ordering::Relaxed); + } else { + metrics.body_size_gt_5mb.fetch_add(1, Ordering::Relaxed); + } +} + +fn observe_worker_processing(metrics: &Metrics, elapsed_ms: u64) { + metrics.worker_processing_count.fetch_add(1, Ordering::Relaxed); + metrics.worker_processing_sum_ms.fetch_add(elapsed_ms, Ordering::Relaxed); + if elapsed_ms <= 1000 { + metrics.worker_processing_le_1000_ms.fetch_add(1, Ordering::Relaxed); + } else if elapsed_ms <= 5000 { + metrics.worker_processing_le_5000_ms.fetch_add(1, Ordering::Relaxed); + } else if elapsed_ms <= 30_000 { + metrics.worker_processing_le_30000_ms.fetch_add(1, Ordering::Relaxed); + } else { + metrics.worker_processing_gt_30000_ms.fetch_add(1, Ordering::Relaxed); + } +} + diff --git a/binary/ai-gateway-service/src/app/object_store.rs b/binary/ai-gateway-service/src/app/object_store.rs new file mode 100644 index 00000000..b62dd101 --- /dev/null +++ b/binary/ai-gateway-service/src/app/object_store.rs @@ -0,0 +1,221 @@ +async fn store_body(state: &AppState, job_id: &str, body: Body) -> Result { + let object_ref = format!("{}/{}/body.bin", state.cfg.object_store_prefix.trim_matches('/'), sanitize_key(job_id)); + let mut stream = body.into_data_stream(); + let mut pending = Vec::new(); + let mut total_size = 0usize; + let mut upload_id = None; + let mut parts = Vec::new(); + let part_size = state.cfg.object_multipart_part_size.max(5 * 1024 * 1024); + + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| ServiceError::bad_request(format!("read request body: {e}")))?; + total_size = total_size.checked_add(chunk.len()).ok_or_else(|| ServiceError::payload_too_large("request body is too large"))?; + if total_size > state.cfg.max_body_bytes { + abort_upload_if_needed(state, &object_ref, upload_id.as_deref()).await; + return Err(ServiceError::payload_too_large(format!("request body exceeds max size {}", state.cfg.max_body_bytes))); + } + + if upload_id.is_none() { + if state.cfg.object_store_endpoint.is_some() && pending.len() + chunk.len() > state.cfg.inline_threshold { + pending.extend_from_slice(&chunk); + match initiate_multipart_upload(state, &object_ref).await { + Ok(id) => upload_id = Some(id), + Err(e) => return Err(e), + } + } else { + pending.extend_from_slice(&chunk); + continue; + } + } else { + pending.extend_from_slice(&chunk); + } + + if let Some(upload_id) = upload_id.as_deref() { + while pending.len() >= part_size { + let part_body = pending.drain(..part_size).collect::>(); + match upload_multipart_part(state, &object_ref, upload_id, parts.len() + 1, part_body).await { + Ok(part) => parts.push(part), + Err(e) => { + abort_upload_if_needed(state, &object_ref, Some(upload_id)).await; + return Err(e); + } + } + } + } + } + + if let Some(upload_id) = upload_id.as_deref() { + if !pending.is_empty() || parts.is_empty() { + match upload_multipart_part(state, &object_ref, upload_id, parts.len() + 1, pending).await { + Ok(part) => parts.push(part), + Err(e) => { + abort_upload_if_needed(state, &object_ref, Some(upload_id)).await; + return Err(e); + } + } + } + if let Err(e) = complete_multipart_upload(state, &object_ref, upload_id, &parts).await { + abort_upload_if_needed(state, &object_ref, Some(upload_id)).await; + return Err(e); + } + state.metrics.object_offload_total.fetch_add(1, Ordering::Relaxed); + return Ok(BodyLocation { + body_base64: String::new(), + object_ref, + size: total_size, + storage: "object", + }); + } + + Ok(BodyLocation { + body_base64: base64::engine::general_purpose::STANDARD.encode(&pending), + object_ref: String::new(), + size: total_size, + storage: "inline", + }) +} + +async fn load_body(state: &AppState, fields: &HashMap) -> Result, ServiceError> { + let storage = field_string(fields, "storage").unwrap_or_else(|| "inline".to_string()); + if storage == "object" { + let object_ref = field_string(fields, "ref").ok_or_else(|| ServiceError::bad_request("job body is missing object ref"))?; + let url = object_url(state, &object_ref); + let mut req = state.http.get(url); + if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { + req = req.header(name, value); + } + return Ok(req.send().await?.error_for_status()?.bytes().await?.to_vec()); + } + + if let Some(body_base64) = field_string(fields, "body") { + return base64::engine::general_purpose::STANDARD.decode(body_base64).map_err(|e| ServiceError::bad_request(format!("decode job body: {e}"))); + } + Ok(field_bytes(fields, "body").unwrap_or_default()) +} + +async fn initiate_multipart_upload(state: &AppState, object_ref: &str) -> Result { + let url = object_url_with_query(state, object_ref, "uploads"); + let mut req = state.http.post(url); + if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { + req = req.header(name, value); + } + let body = req.send().await?.error_for_status()?.text().await?; + extract_xml_tag(&body, "UploadId").ok_or_else(|| ServiceError::internal("multipart initiate response missing UploadId")) +} + +async fn upload_multipart_part(state: &AppState, object_ref: &str, upload_id: &str, part_number: usize, body: Vec) -> Result { + let query = format!("partNumber={part_number}&uploadId={}", encode_query_component(upload_id)); + let url = object_url_with_query(state, object_ref, &query); + let mut req = state.http.put(url).body(body); + if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { + req = req.header(name, value); + } + let resp = req.send().await?.error_for_status()?; + let etag = resp + .headers() + .get("etag") + .and_then(|value| value.to_str().ok()) + .map(ToOwned::to_owned) + .ok_or_else(|| ServiceError::internal("multipart upload part response missing ETag"))?; + Ok(CompletedPart { part_number, etag }) +} + +async fn complete_multipart_upload(state: &AppState, object_ref: &str, upload_id: &str, parts: &[CompletedPart]) -> Result<(), ServiceError> { + let query = format!("uploadId={}", encode_query_component(upload_id)); + let url = object_url_with_query(state, object_ref, &query); + let body = complete_multipart_xml(parts); + let mut req = state.http.post(url).header("content-type", "application/xml").body(body); + if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { + req = req.header(name, value); + } + req.send().await?.error_for_status()?; + Ok(()) +} + +async fn abort_multipart_upload(state: &AppState, object_ref: &str, upload_id: &str) -> Result<(), ServiceError> { + let query = format!("uploadId={}", encode_query_component(upload_id)); + let url = object_url_with_query(state, object_ref, &query); + let mut req = state.http.delete(url); + if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { + req = req.header(name, value); + } + req.send().await?.error_for_status()?; + Ok(()) +} + +async fn abort_upload_if_needed(state: &AppState, object_ref: &str, upload_id: Option<&str>) { + let Some(upload_id) = upload_id else { + return; + }; + state.metrics.object_multipart_abort_total.fetch_add(1, Ordering::Relaxed); + if let Err(abort_err) = abort_multipart_upload(state, object_ref, upload_id).await { + tracing::warn!(object_ref = %object_ref, upload_id = %upload_id, error = %abort_err.message, "multipart upload abort failed"); + } +} + +fn complete_multipart_xml(parts: &[CompletedPart]) -> String { + let mut out = String::from(""); + for part in parts { + out.push_str(""); + out.push_str(""); + out.push_str(&part.part_number.to_string()); + out.push_str(""); + out.push_str(""); + out.push_str(&xml_escape(&part.etag)); + out.push_str(""); + out.push_str(""); + } + out.push_str(""); + out +} + +fn object_url(state: &AppState, object_ref: &str) -> String { + format!( + "{}/{}/{}", + state.cfg.object_store_endpoint.as_deref().unwrap_or_default().trim_end_matches('/'), + state.cfg.object_store_bucket.trim_matches('/'), + object_ref.trim_start_matches('/') + ) +} + +fn object_url_with_query(state: &AppState, object_ref: &str, query: &str) -> String { + format!("{}?{}", object_url(state, object_ref), query) +} + +fn object_auth_header(raw: &Option) -> Result, ServiceError> { + let Some(raw) = raw.as_deref() else { + return Ok(None); + }; + let Some((name, value)) = raw.split_once(':') else { + return Err(ServiceError::bad_request("AI_OBJECT_STORE_AUTH_HEADER must be `Header-Name: value`")); + }; + if HeaderName::try_from(name.trim()).is_err() || HeaderValue::from_str(value.trim()).is_err() { + return Err(ServiceError::bad_request("invalid object auth header")); + } + Ok(Some((name.trim().to_string(), value.trim().to_string()))) +} + +fn extract_xml_tag(xml: &str, tag: &str) -> Option { + let start_tag = format!("<{tag}>"); + let end_tag = format!(""); + let start = xml.find(&start_tag)? + start_tag.len(); + let end = xml[start..].find(&end_tag)? + start; + Some(xml[start..end].trim().to_string()) +} + +fn encode_query_component(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + for byte in input.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + out.push(byte as char); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + +fn xml_escape(input: &str) -> String { + input.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """).replace('\'', "'") +} + diff --git a/binary/ai-gateway-service/src/app/queue.rs b/binary/ai-gateway-service/src/app/queue.rs new file mode 100644 index 00000000..2c2b378d --- /dev/null +++ b/binary/ai-gateway-service/src/app/queue.rs @@ -0,0 +1,437 @@ +async fn enqueue_job(state: &AppState, policy: QueuePolicy, _method: Method, uri: Uri, headers: HeaderMap, body: Body) -> Result { + let enqueue_started_at = now_ms(); + let _permit = state.body_permits.acquire().await.map_err(|_| ServiceError::internal("body semaphore closed"))?; + let job_id = new_job_id(); + let tenant_id = required_header(&headers, "x-tenant-id")?; + let model = optional_header(&headers, "x-model").unwrap_or_else(|| "default".to_string()); + let callback_url = optional_header(&headers, "x-callback-url").unwrap_or_default(); + validate_callback_url(state, policy, &callback_url)?; + let original_method = optional_header(&headers, "x-original-method").unwrap_or_else(|| "POST".to_string()); + let original_path = optional_header(&headers, "x-original-path").unwrap_or_else(|| uri.path().to_string()); + let request_headers = headers_to_json(&headers)?; + let created_at = now_ms(); + let body_ref = store_body(state, &job_id, body).await?; + let (stream_key, priority) = stream_for_request(state, &headers, &tenant_id, &model); + + let stream_id: String = state + .redis + .xadd( + stream_key.as_str(), + false, + None::<()>, + "*", + vec![ + ("job_id", Value::String(job_id.clone().into())), + ("tenant_id", Value::String(tenant_id.into())), + ("policy", Value::String(policy.as_str().into())), + ("model", Value::String(model.into())), + ("priority", Value::String(priority.as_str().into())), + ("method", Value::String(original_method.into())), + ("path", Value::String(original_path.into())), + ("headers", Value::String(request_headers.into())), + ("body", Value::String(body_ref.body_base64.into())), + ("ref", Value::String(body_ref.object_ref.into())), + ("size", Value::Integer(body_ref.size as i64)), + ("storage", Value::String(body_ref.storage.into())), + ("callback_url", Value::String(callback_url.into())), + ("created_at", Value::Integer(created_at as i64)), + ], + ) + .await?; + trim_stream(state, &stream_key).await?; + + state.metrics.enqueue_total.fetch_add(1, Ordering::Relaxed); + observe_enqueue_latency(&state.metrics, now_ms().saturating_sub(enqueue_started_at)); + observe_body_size(&state.metrics, body_ref.size); + match priority { + QueuePriority::High => { + state.metrics.enqueue_priority_high_total.fetch_add(1, Ordering::Relaxed); + } + QueuePriority::Normal => { + state.metrics.enqueue_priority_normal_total.fetch_add(1, Ordering::Relaxed); + } + QueuePriority::Low => { + state.metrics.enqueue_priority_low_total.fetch_add(1, Ordering::Relaxed); + } + } + match policy { + QueuePolicy::Queue => { + state.metrics.enqueue_queue_total.fetch_add(1, Ordering::Relaxed); + } + QueuePolicy::Wait => { + state.metrics.enqueue_wait_total.fetch_add(1, Ordering::Relaxed); + } + } + + Ok(AcceptedJob { + response: EnqueueResponse { + job_id: job_id.clone(), + stream_id, + stream_key, + status: "queued", + poll_url: format!("/v1/jobs/{job_id}"), + }, + created_at_ms: created_at, + }) +} + +fn spawn_workers(state: AppState) { + for idx in 0..state.cfg.worker_concurrency.max(1) { + let state = state.clone(); + tokio::spawn(async move { + let consumer = format!("{}-{idx}", state.cfg.consumer_name); + loop { + if let Err(e) = worker_once(&state, &consumer).await { + tracing::warn!(error = %e.message, "worker loop failed"); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + }); + } +} + +async fn worker_once(state: &AppState, consumer: &str) -> Result<(), ServiceError> { + let streams = worker_stream_order(state); + for (idx, stream) in streams.iter().enumerate() { + let block = if idx + 1 == streams.len() { 1000 } else { 10 }; + let processed = read_worker_stream(state, consumer, stream, block).await?; + if processed > 0 { + return Ok(()); + } + } + Ok(()) +} + +async fn read_worker_stream(state: &AppState, consumer: &str, stream: &str, block_ms: u64) -> Result { + let reply: XReadResponse = + state.redis.xreadgroup_map(state.cfg.consumer_group.as_str(), consumer, Some(5), Some(block_ms), false, vec![stream], vec![">"]).await?; + + let mut processed = 0; + for (_stream, entries) in reply { + for (entry_id, fields) in entries { + match process_stream_entry(state, stream, entry_id.as_str(), &fields).await { + Ok(true) => { + processed += 1; + } + Ok(false) => {} + Err(e) => { + tracing::warn!(stream_id = %entry_id, error = %e.message, "job processing failed"); + state.metrics.worker_failed_total.fetch_add(1, Ordering::Relaxed); + } + } + } + } + Ok(processed) +} + +async fn process_stream_entry(state: &AppState, stream: &str, entry_id: &str, fields: &HashMap) -> Result { + let job_id = field_string(fields, "job_id").ok_or_else(|| ServiceError::bad_request("job missing job_id"))?; + let lease_owner = format!("{}:{stream}:{entry_id}:{}", state.cfg.consumer_name, now_ms()); + + if !acquire_job_lease(state, &job_id, &lease_owner).await? { + state.metrics.lease_skip_total.fetch_add(1, Ordering::Relaxed); + tracing::info!(job_id = %job_id, stream = %stream, entry_id = %entry_id, "job is already leased; skip reclaimed duplicate"); + return Ok(false); + } + + let attempt = increment_job_delivery_attempt(state, &job_id).await?; + if attempt > state.cfg.job_max_delivery_attempts { + enqueue_job_dlq(state, stream, entry_id, fields, attempt, "max_delivery_attempts_exceeded").await?; + ack_stream_entry(state, stream, entry_id).await?; + release_job_lease(state, &job_id).await; + state.metrics.job_dlq_total.fetch_add(1, Ordering::Relaxed); + return Ok(true); + } + + let processing_started_at = now_ms(); + match process_job(state, stream, entry_id, fields).await { + Ok(()) => { + observe_worker_processing(&state.metrics, now_ms().saturating_sub(processing_started_at)); + ack_stream_entry(state, stream, entry_id).await?; + clear_job_delivery_attempt(state, &job_id).await; + release_job_lease(state, &job_id).await; + Ok(true) + } + Err(e) => { + observe_worker_processing(&state.metrics, now_ms().saturating_sub(processing_started_at)); + release_job_lease(state, &job_id).await; + Err(e) + } + } +} + +async fn process_job(state: &AppState, _stream: &str, _stream_id: &str, fields: &HashMap) -> Result<(), ServiceError> { + let Some(base) = state.cfg.upstream_base_url.as_deref() else { + return Err(ServiceError::internal("upstream base URL is not configured")); + }; + let job_id = field_string(fields, "job_id").ok_or_else(|| ServiceError::bad_request("job missing job_id"))?; + let method = field_string(fields, "method").unwrap_or_else(|| "POST".to_string()); + let path = field_string(fields, "path").unwrap_or_else(|| "/".to_string()); + let headers_json = field_string(fields, "headers").unwrap_or_else(|| "{}".to_string()); + let callback_url = field_string(fields, "callback_url").unwrap_or_default(); + let body = load_body(state, fields).await?; + let headers: HashMap = serde_json::from_str(&headers_json).unwrap_or_default(); + + let url = format!("{}{}", base.trim_end_matches('/'), path); + let parsed_method = method.parse::().unwrap_or(reqwest::Method::POST); + let mut req = state.http.request(parsed_method, url); + for (name, value) in headers { + if should_forward_header(&name) { + req = req.header(name, value); + } + } + let upstream = req.body(body).send().await; + let result = match upstream { + Ok(resp) => { + let status = resp.status().as_u16(); + let mut headers = HashMap::new(); + for (name, value) in resp.headers() { + if let Ok(value) = value.to_str() { + headers.insert(name.as_str().to_string(), value.to_string()); + } + } + let body = resp.bytes().await.unwrap_or_default(); + StoredResult { + job_id: job_id.clone(), + status: "completed".to_string(), + http_status: status, + headers, + body_base64: base64::engine::general_purpose::STANDARD.encode(body), + completed_at_ms: now_ms(), + error: None, + } + } + Err(e) => StoredResult { + job_id: job_id.clone(), + status: "failed".to_string(), + http_status: 502, + headers: HashMap::new(), + body_base64: String::new(), + completed_at_ms: now_ms(), + error: Some(e.to_string()), + }, + }; + + store_result(state, &result).await?; + if !callback_url.is_empty() { + let callback_body = callback_body(&result); + if let Err(e) = post_callback(state, &callback_url, &job_id, &callback_body).await { + tracing::warn!(job_id = %job_id, error = %e.message, "callback failed"); + state.metrics.callback_failure_total.fetch_add(1, Ordering::Relaxed); + enqueue_callback_retry(state, &callback_url, &job_id, &callback_body, e.message.as_str()).await?; + } + } + state.metrics.worker_completed_total.fetch_add(1, Ordering::Relaxed); + Ok(()) +} + +async fn acquire_job_lease(state: &AppState, job_id: &str, owner: &str) -> Result { + let key = job_lease_key(job_id); + let result: Option = state + .redis + .set( + key, + owner, + Some(Expiration::EX(state.cfg.job_process_lease_secs.max(1) as i64)), + Some(SetOptions::NX), + false, + ) + .await?; + Ok(result.is_some()) +} + +async fn release_job_lease(state: &AppState, job_id: &str) { + let _: Result = state.redis.del(job_lease_key(job_id)).await; +} + +async fn increment_job_delivery_attempt(state: &AppState, job_id: &str) -> Result { + let key = job_attempt_key(job_id); + let attempt: i64 = state.redis.incr_by(key.as_str(), 1).await?; + let _: () = state.redis.expire(key.as_str(), state.cfg.result_ttl_secs.max(300) as i64, None::).await?; + Ok(attempt.max(0) as u32) +} + +async fn clear_job_delivery_attempt(state: &AppState, job_id: &str) { + let _: Result = state.redis.del(job_attempt_key(job_id)).await; +} + +async fn ack_stream_entry(state: &AppState, stream: &str, entry_id: &str) -> Result<(), ServiceError> { + let _: i64 = state.redis.xack(stream, state.cfg.consumer_group.as_str(), vec![entry_id]).await?; + Ok(()) +} + +async fn enqueue_job_dlq(state: &AppState, stream: &str, entry_id: &str, fields: &HashMap, attempts: u32, reason: &str) -> Result<(), ServiceError> { + let job_id = field_string(fields, "job_id").unwrap_or_default(); + let fields_json = stream_fields_to_json(fields)?; + let _: String = state + .redis + .xadd( + state.cfg.job_dlq_stream.as_str(), + false, + None::<()>, + "*", + vec![ + ("job_id", Value::String(job_id.into())), + ("source_stream", Value::String(stream.to_string().into())), + ("source_entry_id", Value::String(entry_id.to_string().into())), + ("attempts", Value::Integer(attempts as i64)), + ("reason", Value::String(reason.to_string().into())), + ("fields", Value::String(fields_json.into())), + ("failed_at", Value::Integer(now_ms() as i64)), + ], + ) + .await?; + trim_stream(state, &state.cfg.job_dlq_stream).await?; + Ok(()) +} + +fn stream_fields_to_json(fields: &HashMap) -> Result { + let mut out = HashMap::new(); + for (key, value) in fields { + if let Some(value) = field_string(fields, key) { + out.insert(key.clone(), value); + } else { + out.insert(key.clone(), format!("{value:?}")); + } + } + serde_json::to_string(&out).map_err(|e| ServiceError::internal(format!("serialize job dlq fields: {e}"))) +} + +fn job_lease_key(job_id: &str) -> String { + format!("ai:job:lease:{}", sanitize_key(job_id)) +} + +fn job_attempt_key(job_id: &str) -> String { + format!("ai:job:attempt:{}", sanitize_key(job_id)) +} + +fn spawn_reclaimer(state: AppState) { + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(state.cfg.reclaim_interval_secs.max(1))); + loop { + interval.tick().await; + if let Err(e) = reclaim_once(&state).await { + tracing::warn!(error = %e.message, "stream reclaim failed"); + } + } + }); +} + +async fn reclaim_once(state: &AppState) -> Result<(), ServiceError> { + let consumer = format!("{}-reclaimer", state.cfg.consumer_name); + let min_idle_ms = state.cfg.reclaim_min_idle_secs.saturating_mul(1000); + for stream in configured_streams(state) { + let (_cursor, entries): (String, Vec<(String, HashMap)>) = + state.redis.xautoclaim_values(stream.as_str(), state.cfg.consumer_group.as_str(), consumer.as_str(), min_idle_ms, "0-0", Some(10), false).await?; + for (entry_id, fields) in entries { + match process_stream_entry(state, stream.as_str(), entry_id.as_str(), &fields).await { + Ok(true) => { + state.metrics.reclaimed_total.fetch_add(1, Ordering::Relaxed); + } + Ok(false) => {} + Err(e) => { + tracing::warn!(stream = %stream, entry_id = %entry_id, error = %e.message, "reclaimed job failed"); + } + } + } + } + Ok(()) +} + +async fn ensure_consumer_groups(state: &AppState) -> Result<(), ServiceError> { + for stream in configured_streams(state) { + ensure_consumer_group(state, &stream, &state.cfg.consumer_group).await?; + } + ensure_consumer_group(state, &state.cfg.callback_retry_stream, &state.cfg.callback_retry_group).await?; + Ok(()) +} + +async fn ensure_consumer_group(state: &AppState, stream: &str, group: &str) -> Result<(), ServiceError> { + let res: FredResult = state.redis.xgroup_create(stream, group, "$", true).await; + match res { + Ok(_) => Ok(()), + Err(e) if e.to_string().contains("BUSYGROUP") => Ok(()), + Err(e) => Err(e.into()), + } +} + +fn stream_for_request(state: &AppState, headers: &HeaderMap, tenant: &str, model: &str) -> (String, QueuePriority) { + let priority = request_priority(state, headers, tenant, model); + if !state.cfg.enable_priority_streams { + return (state.cfg.stream_key.clone(), priority); + } + let stream = match priority { + QueuePriority::High => state.cfg.high_priority_stream_key.clone(), + QueuePriority::Low => state.cfg.low_priority_stream_key.clone(), + QueuePriority::Normal => state.cfg.stream_key.clone(), + }; + (stream, priority) +} + +fn request_priority(state: &AppState, headers: &HeaderMap, tenant: &str, model: &str) -> QueuePriority { + if let Some(priority) = optional_header(headers, "x-queue-priority").and_then(|value| parse_queue_priority(&value)) { + return priority; + } + if contains_csv_value(&state.cfg.queue_high_tenants, tenant) || contains_csv_value(&state.cfg.queue_high_models, model) { + return QueuePriority::High; + } + if contains_csv_value(&state.cfg.queue_low_tenants, tenant) || contains_csv_value(&state.cfg.queue_low_models, model) { + return QueuePriority::Low; + } + parse_queue_priority(&state.cfg.queue_default_priority).unwrap_or(QueuePriority::Normal) +} + +fn configured_streams(state: &AppState) -> Vec { + if state.cfg.enable_priority_streams { + vec![ + state.cfg.high_priority_stream_key.clone(), + state.cfg.stream_key.clone(), + state.cfg.low_priority_stream_key.clone(), + ] + } else { + vec![state.cfg.stream_key.clone()] + } +} + +fn worker_stream_order(state: &AppState) -> Vec { + if !state.cfg.enable_priority_streams { + return vec![state.cfg.stream_key.clone()]; + } + let mut out = Vec::new(); + push_weighted(&mut out, &state.cfg.high_priority_stream_key, state.cfg.queue_high_weight); + push_weighted(&mut out, &state.cfg.stream_key, state.cfg.queue_normal_weight); + push_weighted(&mut out, &state.cfg.low_priority_stream_key, state.cfg.queue_low_weight); + if out.is_empty() { + out.push(state.cfg.stream_key.clone()); + } + out +} + +fn push_weighted(out: &mut Vec, stream: &str, weight: usize) { + for _ in 0..weight { + out.push(stream.to_string()); + } +} + +fn parse_queue_priority(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "high" => Some(QueuePriority::High), + "normal" | "default" | "medium" => Some(QueuePriority::Normal), + "low" => Some(QueuePriority::Low), + _ => None, + } +} + +fn contains_csv_value(csv: &str, needle: &str) -> bool { + csv.split(',').map(str::trim).filter(|value| !value.is_empty()).any(|value| value.eq_ignore_ascii_case(needle)) +} + +fn validate_callback_url(state: &AppState, policy: QueuePolicy, callback_url: &str) -> Result<(), ServiceError> { + if policy == QueuePolicy::Queue && callback_url.is_empty() { + return Err(ServiceError::bad_request("missing required header `x-callback-url` for queue policy")); + } + if !callback_url.is_empty() && state.cfg.require_https_callback && !callback_url.starts_with("https://") { + return Err(ServiceError::bad_request("x-callback-url must use https")); + } + Ok(()) +} diff --git a/binary/ai-gateway-service/src/app/ratelimit.rs b/binary/ai-gateway-service/src/app/ratelimit.rs new file mode 100644 index 00000000..48ed9535 --- /dev/null +++ b/binary/ai-gateway-service/src/app/ratelimit.rs @@ -0,0 +1,52 @@ +async fn tenant_rate_limit(state: &AppState, tenant: &str, model: &str, path: &str, policy: &str) -> Result { + for key in tenant_rate_limit_candidate_keys(state, tenant, model, path, policy) { + let raw: Option = state.redis.get(key.as_str()).await.unwrap_or(None); + if let Some(limit) = raw.and_then(|raw| parse_tenant_rate_limit(&raw)) { + return Ok(limit); + } + } + + let key = format!("{}{}", state.cfg.tenant_rate_limit_prefix, sanitize_key(tenant)); + let rps: Option = state.redis.get(format!("{key}:rps")).await.unwrap_or(None); + let burst: Option = state.redis.get(format!("{key}:burst")).await.unwrap_or(None); + let cost: Option = state.redis.get(format!("{key}:cost")).await.unwrap_or(None); + Ok(TenantRateLimit { + rps: rps.and_then(|v| v.parse().ok()).unwrap_or(state.cfg.rate_limit_rps), + burst: burst.and_then(|v| v.parse().ok()).unwrap_or(state.cfg.rate_limit_burst), + cost: cost.and_then(|v| v.parse().ok()).unwrap_or(state.cfg.rate_limit_cost).max(1), + }) +} + +fn tenant_rate_limit_candidate_keys(state: &AppState, tenant: &str, model: &str, path: &str, policy: &str) -> Vec { + let base = format!("{}{}", state.cfg.tenant_rate_limit_prefix, sanitize_key(tenant)); + let model = sanitize_key(model); + let path = sanitize_key(path); + let policy = sanitize_key(policy); + vec![ + format!("{base}:model:{model}:path:{path}:policy:{policy}"), + format!("{base}:model:{model}:path:{path}"), + format!("{base}:model:{model}:policy:{policy}"), + format!("{base}:path:{path}:policy:{policy}"), + format!("{base}:model:{model}"), + format!("{base}:path:{path}"), + format!("{base}:policy:{policy}"), + base, + ] +} + +fn parse_tenant_rate_limit(raw: &str) -> Option { + let raw = raw.trim(); + if raw.is_empty() { + return None; + } + if let Ok(mut limit) = serde_json::from_str::(raw) { + limit.cost = limit.cost.max(1); + return Some(limit); + } + + let mut parts = raw.split(',').map(str::trim); + let rps = parts.next()?.parse().ok()?; + let burst = parts.next()?.parse().ok()?; + let cost = parts.next().and_then(|value| value.parse().ok()).unwrap_or(1); + Some(TenantRateLimit { rps, burst, cost: cost.max(1) }) +} diff --git a/binary/ai-gateway-service/src/app/result_store.rs b/binary/ai-gateway-service/src/app/result_store.rs new file mode 100644 index 00000000..6f1b886f --- /dev/null +++ b/binary/ai-gateway-service/src/app/result_store.rs @@ -0,0 +1,28 @@ +async fn store_result(state: &AppState, result: &StoredResult) -> Result<(), ServiceError> { + let json = serde_json::to_string(result).map_err(|e| ServiceError::internal(format!("serialize result: {e}")))?; + let key = result_key(state, &result.job_id); + let channel = result_channel(state, &result.job_id); + let ttl = state.cfg.result_ttl_secs.min(i64::MAX as u64) as i64; + let _: () = state.redis.set(key, json, Some(Expiration::EX(ttl)), None::, false).await?; + let _: i64 = state.redis.publish(channel, "done").await?; + Ok(()) +} + +async fn load_result(state: &AppState, job_id: &str) -> Result, ServiceError> { + let raw: Option = state.redis.get(result_key(state, job_id)).await?; + raw.map(|s| serde_json::from_str(&s).map_err(|e| ServiceError::internal(format!("parse result: {e}")))).transpose() +} + +fn result_to_response(result: StoredResult, created_at_ms: u64) -> Result { + let status = StatusCode::from_u16(result.http_status).unwrap_or(StatusCode::OK); + let body = base64::engine::general_purpose::STANDARD.decode(result.body_base64).map_err(|e| ServiceError::internal(format!("decode result body: {e}")))?; + let mut resp = (status, body).into_response(); + for (name, value) in result.headers { + if let (Ok(name), Ok(value)) = (HeaderName::try_from(name.as_str()), HeaderValue::from_str(&value)) { + resp.headers_mut().insert(name, value); + } + } + resp.headers_mut().insert("x-job-id", header_value(&result.job_id)?); + resp.headers_mut().insert("x-queue-wait-ms", header_value(&now_ms().saturating_sub(created_at_ms).to_string())?); + Ok(resp) +} diff --git a/binary/ai-gateway-service/src/app/runtime.rs b/binary/ai-gateway-service/src/app/runtime.rs new file mode 100644 index 00000000..cdee8851 --- /dev/null +++ b/binary/ai-gateway-service/src/app/runtime.rs @@ -0,0 +1,40 @@ +pub async fn run() -> Result<(), Box> { + tracing_subscriber::fmt().with_env_filter(tracing_subscriber::EnvFilter::from_default_env()).init(); + + let args = Args::parse(); + let redis = build_redis_client(&args.redis_url)?; + let _redis_task = redis.init().await?; + let state = AppState { + redis, + http: reqwest::Client::new(), + cfg: Arc::new(args.clone()), + body_permits: Arc::new(Semaphore::new(args.body_read_concurrency.max(1))), + metrics: Arc::new(Metrics::default()), + }; + + ensure_consumer_groups(&state).await?; + if state.cfg.upstream_base_url.is_some() { + spawn_workers(state.clone()); + spawn_reclaimer(state.clone()); + spawn_callback_retry_worker(state.clone()); + } else { + tracing::warn!("AI_UPSTREAM_BASE_URL is not set; queue jobs will be stored but no local worker will process them"); + } + + let app = Router::new() + .route("/healthz", get(healthz)) + .route("/metrics", get(metrics)) + .route("/v1/ratelimit/check", post(check_rate_limit)) + .route("/v1/queue/enqueue", post(enqueue)) + .route("/v1/queue/enqueue-and-wait", post(enqueue_and_wait)) + .route("/v1/jobs/{job_id}", get(get_job)) + .layer(DefaultBodyLimit::max(args.max_body_bytes)) + .layer(TraceLayer::new_for_http()) + .with_state(state); + + let addr = SocketAddr::new(args.host, args.port); + tracing::info!(%addr, "ai-gateway-service listening"); + let listener = tokio::net::TcpListener::bind(addr).await?; + axum::serve(listener, app).await?; + Ok(()) +} diff --git a/binary/ai-gateway-service/src/app/tests.rs b/binary/ai-gateway-service/src/app/tests.rs new file mode 100644 index 00000000..421913a1 --- /dev/null +++ b/binary/ai-gateway-service/src/app/tests.rs @@ -0,0 +1,85 @@ +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extracts_upload_id_from_multipart_xml() { + let xml = "a+b/c="; + assert_eq!(extract_xml_tag(xml, "UploadId").as_deref(), Some("a+b/c=")); + } + + #[test] + fn encodes_upload_id_for_query_string() { + assert_eq!(encode_query_component("a+b/c="), "a%2Bb%2Fc%3D"); + } + + #[test] + fn builds_complete_multipart_xml_with_escaped_etags() { + let parts = vec![ + CompletedPart { + part_number: 1, + etag: "\"abc&1\"".to_string(), + }, + CompletedPart { + part_number: 2, + etag: "\"def\"".to_string(), + }, + ]; + let xml = complete_multipart_xml(&parts); + assert!(xml.contains("1"abc&1"")); + assert!(xml.contains("2"def"")); + } + + #[test] + fn callback_retry_delay_uses_exponential_backoff_with_cap() { + assert_eq!(callback_retry_delay_ms(1000, 60_000, 1), 1000); + assert_eq!(callback_retry_delay_ms(1000, 60_000, 3), 4000); + assert_eq!(callback_retry_delay_ms(1000, 5000, 8), 5000); + } + + #[test] + fn parses_xpending_summary_count() { + let value = Value::Array(vec![Value::Integer(7), Value::String("0-1".into()), Value::String("0-2".into())]); + assert_eq!(pending_count_from_value(&value), 7); + } + + #[test] + fn observes_histogram_buckets_as_non_overlapping_counts() { + let metrics = Metrics::default(); + observe_enqueue_latency(&metrics, 80); + observe_enqueue_latency(&metrics, 800); + observe_body_size(&metrics, 8 * 1024); + observe_body_size(&metrics, 256 * 1024); + observe_worker_processing(&metrics, 2000); + + assert_eq!(metrics.enqueue_latency_count.load(Ordering::Relaxed), 2); + assert_eq!(metrics.enqueue_latency_le_100_ms.load(Ordering::Relaxed), 1); + assert_eq!(metrics.enqueue_latency_le_1000_ms.load(Ordering::Relaxed), 1); + assert_eq!(metrics.body_size_count.load(Ordering::Relaxed), 2); + assert_eq!(metrics.body_size_le_10kb.load(Ordering::Relaxed), 1); + assert_eq!(metrics.body_size_le_5mb.load(Ordering::Relaxed), 1); + assert_eq!(metrics.worker_processing_count.load(Ordering::Relaxed), 1); + assert_eq!(metrics.worker_processing_le_5000_ms.load(Ordering::Relaxed), 1); + } + + #[test] + fn parses_tenant_rate_limit_json_and_csv() { + let json = parse_tenant_rate_limit(r#"{"rps":10,"burst":20,"cost":3}"#).unwrap(); + assert_eq!(json.rps, 10); + assert_eq!(json.burst, 20); + assert_eq!(json.cost, 3); + + let csv = parse_tenant_rate_limit("15,30,2").unwrap(); + assert_eq!(csv.rps, 15); + assert_eq!(csv.burst, 30); + assert_eq!(csv.cost, 2); + } + + #[test] + fn parses_queue_priority_values() { + assert_eq!(parse_queue_priority("HIGH"), Some(QueuePriority::High)); + assert_eq!(parse_queue_priority("medium"), Some(QueuePriority::Normal)); + assert_eq!(parse_queue_priority("low"), Some(QueuePriority::Low)); + assert_eq!(parse_queue_priority("urgent"), None); + } +} diff --git a/binary/ai-gateway-service/src/app/types.rs b/binary/ai-gateway-service/src/app/types.rs new file mode 100644 index 00000000..b29ff26f --- /dev/null +++ b/binary/ai-gateway-service/src/app/types.rs @@ -0,0 +1,342 @@ +static JOB_COUNTER: AtomicU64 = AtomicU64::new(1); + +const TOKEN_BUCKET_LUA: &str = r#" +local tokens_key = KEYS[1] +local ts_key = KEYS[2] +local rate = tonumber(ARGV[1]) +local burst = tonumber(ARGV[2]) +local now = tonumber(ARGV[3]) +local cost = tonumber(ARGV[4]) + +if rate <= 0 or burst <= 0 or cost <= 0 then + return {0, 0, 1000} +end + +local burst_milli = burst * 1000 +local cost_milli = cost * 1000 +local tokens = tonumber(redis.call('GET', tokens_key) or burst_milli) +local last_ts = tonumber(redis.call('GET', ts_key) or now) +local elapsed = math.max(0, now - last_ts) +tokens = math.min(burst_milli, tokens + elapsed * rate) + +local ttl = math.max(1000, math.ceil((burst_milli / rate) * 2)) +if tokens >= cost_milli then + tokens = tokens - cost_milli + redis.call('SET', tokens_key, tokens, 'PX', ttl) + redis.call('SET', ts_key, now, 'PX', ttl) + return {1, tokens, 0} +else + local wait_ms = math.ceil((cost_milli - tokens) / rate) + redis.call('SET', tokens_key, tokens, 'PX', ttl) + redis.call('SET', ts_key, now, 'PX', ttl) + return {0, tokens, wait_ms} +end +"#; + +#[derive(Debug, Clone, Parser)] +#[command(version, about = "External Redis-backed rate-limit and queue service for SpaceGate AI gateway")] +struct Args { + #[arg(long, env = "AI_GATEWAY_SERVICE_HOST", default_value = "0.0.0.0")] + host: IpAddr, + #[arg(long, env = "AI_GATEWAY_SERVICE_PORT", default_value_t = 18080)] + port: u16, + #[arg(long, env = "REDIS_URL", default_value = "redis://127.0.0.1/")] + redis_url: String, + #[arg(long, env = "AI_QUEUE_STREAM", default_value = "ai:jobs")] + stream_key: String, + #[arg(long, env = "AI_QUEUE_HIGH_STREAM", default_value = "ai:jobs:high")] + high_priority_stream_key: String, + #[arg(long, env = "AI_QUEUE_LOW_STREAM", default_value = "ai:jobs:low")] + low_priority_stream_key: String, + #[arg(long, env = "AI_ENABLE_PRIORITY_STREAMS", default_value_t = false)] + enable_priority_streams: bool, + #[arg(long, env = "AI_QUEUE_DEFAULT_PRIORITY", default_value = "normal")] + queue_default_priority: String, + #[arg(long, env = "AI_QUEUE_HIGH_MODELS", default_value = "")] + queue_high_models: String, + #[arg(long, env = "AI_QUEUE_LOW_MODELS", default_value = "")] + queue_low_models: String, + #[arg(long, env = "AI_QUEUE_HIGH_TENANTS", default_value = "")] + queue_high_tenants: String, + #[arg(long, env = "AI_QUEUE_LOW_TENANTS", default_value = "")] + queue_low_tenants: String, + #[arg(long, env = "AI_QUEUE_HIGH_WEIGHT", default_value_t = 3)] + queue_high_weight: usize, + #[arg(long, env = "AI_QUEUE_NORMAL_WEIGHT", default_value_t = 1)] + queue_normal_weight: usize, + #[arg(long, env = "AI_QUEUE_LOW_WEIGHT", default_value_t = 1)] + queue_low_weight: usize, + #[arg(long, env = "AI_QUEUE_MAX_LEN", default_value_t = 100_000)] + stream_max_len: u64, + #[arg(long, env = "AI_QUEUE_GROUP", default_value = "ai-gateway-workers")] + consumer_group: String, + #[arg(long, env = "AI_QUEUE_CONSUMER", default_value = "ai-gateway-service")] + consumer_name: String, + #[arg(long, env = "AI_JOB_DLQ_STREAM", default_value = "ai:job-dlq")] + job_dlq_stream: String, + #[arg(long, env = "AI_CALLBACK_RETRY_STREAM", default_value = "ai:callback-retry")] + callback_retry_stream: String, + #[arg(long, env = "AI_CALLBACK_RETRY_GROUP", default_value = "ai-gateway-callbacks")] + callback_retry_group: String, + #[arg(long, env = "AI_CALLBACK_DLQ_STREAM", default_value = "ai:callback-dlq")] + callback_dlq_stream: String, + #[arg(long, env = "AI_CALLBACK_MAX_RETRY_ATTEMPTS", default_value_t = 5)] + callback_max_retry_attempts: u32, + #[arg(long, env = "AI_CALLBACK_RETRY_INITIAL_DELAY_MS", default_value_t = 1000)] + callback_retry_initial_delay_ms: u64, + #[arg(long, env = "AI_CALLBACK_RETRY_MAX_DELAY_MS", default_value_t = 60_000)] + callback_retry_max_delay_ms: u64, + #[arg(long, env = "AI_CALLBACK_RETRY_RECLAIM_IDLE_SECS", default_value_t = 60)] + callback_retry_reclaim_idle_secs: u64, + #[arg(long, env = "AI_RESULT_KEY_PREFIX", default_value = "result:")] + result_key_prefix: String, + #[arg(long, env = "AI_RESULT_CHANNEL_PREFIX", default_value = "result:")] + result_channel_prefix: String, + #[arg(long, env = "AI_RESULT_TTL_SECS", default_value_t = 120)] + result_ttl_secs: u64, + #[arg(long, env = "AI_RATE_LIMIT_RPS", default_value_t = 100)] + rate_limit_rps: u64, + #[arg(long, env = "AI_RATE_LIMIT_BURST", default_value_t = 200)] + rate_limit_burst: u64, + #[arg(long, env = "AI_RATE_LIMIT_COST", default_value_t = 1)] + rate_limit_cost: u64, + #[arg(long, env = "AI_TENANT_RATE_LIMIT_PREFIX", default_value = "ai:tenant:ratelimit:")] + tenant_rate_limit_prefix: String, + #[arg(long, env = "AI_WAIT_TIMEOUT_SECS", default_value_t = 60)] + wait_timeout_secs: u64, + #[arg(long, env = "AI_WORKER_CONCURRENCY", default_value_t = 1)] + worker_concurrency: usize, + #[arg(long, env = "AI_UPSTREAM_BASE_URL")] + upstream_base_url: Option, + #[arg(long, env = "AI_MAX_BODY_BYTES", default_value_t = 32 * 1024 * 1024)] + max_body_bytes: usize, + #[arg(long, env = "AI_INLINE_THRESHOLD", default_value_t = 128 * 1024)] + inline_threshold: usize, + #[arg(long, env = "AI_BODY_READ_CONCURRENCY", default_value_t = 200)] + body_read_concurrency: usize, + #[arg(long, env = "AI_RECLAIM_INTERVAL_SECS", default_value_t = 30)] + reclaim_interval_secs: u64, + #[arg(long, env = "AI_RECLAIM_MIN_IDLE_SECS", default_value_t = 30)] + reclaim_min_idle_secs: u64, + #[arg(long, env = "AI_JOB_PROCESS_LEASE_SECS", default_value_t = 120)] + job_process_lease_secs: u64, + #[arg(long, env = "AI_JOB_MAX_DELIVERY_ATTEMPTS", default_value_t = 5)] + job_max_delivery_attempts: u32, + #[arg(long, env = "AI_REQUIRE_HTTPS_CALLBACK", default_value_t = true)] + require_https_callback: bool, + #[arg(long, env = "AI_OBJECT_STORE_ENDPOINT")] + object_store_endpoint: Option, + #[arg(long, env = "AI_OBJECT_STORE_BUCKET", default_value = "ai-gateway-body")] + object_store_bucket: String, + #[arg(long, env = "AI_OBJECT_STORE_PREFIX", default_value = "bodies")] + object_store_prefix: String, + #[arg(long, env = "AI_OBJECT_MULTIPART_PART_SIZE", default_value_t = 5 * 1024 * 1024)] + object_multipart_part_size: usize, + #[arg(long, env = "AI_OBJECT_STORE_AUTH_HEADER")] + object_store_auth_header: Option, +} + +#[derive(Clone)] +struct AppState { + redis: FredClient, + http: reqwest::Client, + cfg: Arc, + body_permits: Arc, + metrics: Arc, +} + +#[derive(Default)] +struct Metrics { + rate_limited_total: AtomicU64, + enqueue_total: AtomicU64, + enqueue_queue_total: AtomicU64, + enqueue_wait_total: AtomicU64, + enqueue_priority_high_total: AtomicU64, + enqueue_priority_normal_total: AtomicU64, + enqueue_priority_low_total: AtomicU64, + enqueue_latency_count: AtomicU64, + enqueue_latency_sum_ms: AtomicU64, + enqueue_latency_le_100_ms: AtomicU64, + enqueue_latency_le_500_ms: AtomicU64, + enqueue_latency_le_1000_ms: AtomicU64, + enqueue_latency_gt_1000_ms: AtomicU64, + body_size_le_10kb: AtomicU64, + body_size_le_128kb: AtomicU64, + body_size_le_5mb: AtomicU64, + body_size_gt_5mb: AtomicU64, + body_size_count: AtomicU64, + body_size_sum_bytes: AtomicU64, + wait_total: AtomicU64, + wait_timeout_total: AtomicU64, + callback_failure_total: AtomicU64, + callback_retry_total: AtomicU64, + callback_retry_success_total: AtomicU64, + callback_retry_dlq_total: AtomicU64, + worker_completed_total: AtomicU64, + worker_failed_total: AtomicU64, + worker_processing_count: AtomicU64, + worker_processing_sum_ms: AtomicU64, + worker_processing_le_1000_ms: AtomicU64, + worker_processing_le_5000_ms: AtomicU64, + worker_processing_le_30000_ms: AtomicU64, + worker_processing_gt_30000_ms: AtomicU64, + reclaimed_total: AtomicU64, + job_dlq_total: AtomicU64, + lease_skip_total: AtomicU64, + object_offload_total: AtomicU64, + object_multipart_abort_total: AtomicU64, +} + +#[derive(Debug, Serialize)] +struct RateLimitResponse { + allowed: bool, + remaining_tokens_milli: i64, + retry_after_ms: i64, +} + +#[derive(Debug, Serialize)] +struct EnqueueResponse { + job_id: String, + stream_id: String, + stream_key: String, + status: &'static str, + poll_url: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +struct StoredResult { + job_id: String, + status: String, + http_status: u16, + headers: HashMap, + body_base64: String, + completed_at_ms: u64, + error: Option, +} + +#[derive(Debug)] +struct AcceptedJob { + response: EnqueueResponse, + created_at_ms: u64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum QueuePolicy { + Queue, + Wait, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum QueuePriority { + High, + Normal, + Low, +} + +impl QueuePriority { + fn as_str(self) -> &'static str { + match self { + QueuePriority::High => "high", + QueuePriority::Normal => "normal", + QueuePriority::Low => "low", + } + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +struct TenantRateLimit { + rps: u64, + burst: u64, + #[serde(default = "default_rate_limit_cost")] + cost: u64, +} + +fn default_rate_limit_cost() -> u64 { + 1 +} + +impl QueuePolicy { + fn as_str(self) -> &'static str { + match self { + QueuePolicy::Queue => "queue", + QueuePolicy::Wait => "wait", + } + } +} + +#[derive(Debug)] +struct BodyLocation { + body_base64: String, + object_ref: String, + size: usize, + storage: &'static str, +} + +#[derive(Debug)] +struct CompletedPart { + part_number: usize, + etag: String, +} + +#[derive(Debug)] +struct ServiceError { + status: StatusCode, + message: String, +} + +impl ServiceError { + fn bad_request(message: impl Into) -> Self { + Self { + status: StatusCode::BAD_REQUEST, + message: message.into(), + } + } + + fn internal(message: impl Into) -> Self { + Self { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: message.into(), + } + } + + fn gateway_timeout(message: impl Into) -> Self { + Self { + status: StatusCode::GATEWAY_TIMEOUT, + message: message.into(), + } + } + + fn payload_too_large(message: impl Into) -> Self { + Self { + status: StatusCode::PAYLOAD_TOO_LARGE, + message: message.into(), + } + } +} + +impl IntoResponse for ServiceError { + fn into_response(self) -> Response { + let body = Json(serde_json::json!({ "error": self.message })); + (self.status, body).into_response() + } +} + +impl std::fmt::Display for ServiceError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} + +impl std::error::Error for ServiceError {} + +impl From for ServiceError { + fn from(value: fred::error::Error) -> Self { + Self::internal(format!("redis: {value}")) + } +} + +impl From for ServiceError { + fn from(value: reqwest::Error) -> Self { + Self::internal(format!("http: {value}")) + } +} diff --git a/binary/ai-gateway-service/src/app/util.rs b/binary/ai-gateway-service/src/app/util.rs new file mode 100644 index 00000000..b3f66564 --- /dev/null +++ b/binary/ai-gateway-service/src/app/util.rs @@ -0,0 +1,91 @@ +fn required_header(headers: &HeaderMap, name: &str) -> Result { + optional_header(headers, name).ok_or_else(|| ServiceError::bad_request(format!("missing required header `{name}`"))) +} + +fn optional_header(headers: &HeaderMap, name: &str) -> Option { + headers.get(name).and_then(|value| value.to_str().ok()).map(str::trim).filter(|value| !value.is_empty()).map(ToOwned::to_owned) +} + +fn headers_to_json(headers: &HeaderMap) -> Result { + let mut out = HashMap::new(); + for (name, value) in headers { + if let Ok(value) = value.to_str() { + out.insert(name.as_str().to_string(), value.to_string()); + } + } + serde_json::to_string(&out).map_err(|e| ServiceError::internal(format!("serialize headers: {e}"))) +} + +fn should_forward_header(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + !matches!( + name.as_str(), + "host" | "connection" | "content-length" | "transfer-encoding" | "x-original-method" | "x-original-path" | "x-ratelimit-policy" | "x-callback-url" | "x-request-timeout" + ) +} + +fn header_value(value: &str) -> Result { + HeaderValue::from_str(value).map_err(|e| ServiceError::internal(format!("invalid response header value: {e}"))) +} + +fn result_key(state: &AppState, job_id: &str) -> String { + format!("{}{}", state.cfg.result_key_prefix, job_id) +} + +fn result_channel(state: &AppState, job_id: &str) -> String { + format!("{}{}", state.cfg.result_channel_prefix, job_id) +} + +fn new_job_id() -> String { + let now = now_ms(); + let seq = JOB_COUNTER.fetch_add(1, Ordering::Relaxed); + format!("{now:x}{seq:x}") +} + +fn now_ms() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64 +} + +fn sanitize_key(input: &str) -> String { + input.chars().map(|ch| if ch.is_ascii_alphanumeric() || matches!(ch, ':' | '_' | '-' | '.') { ch } else { '_' }).collect() +} + +fn build_redis_client(url: &str) -> Result { + let config = Config::from_url(url)?; + Builder::from_config(config).build() +} + +fn build_subscriber_client(url: &str) -> Result { + let config = Config::from_url(url)?; + Builder::from_config(config).build_subscriber_client() +} + +fn field_string(fields: &HashMap, key: &str) -> Option { + fields.get(key).and_then(|value| match value { + Value::String(value) => Some(value.to_string()), + Value::Bytes(value) => String::from_utf8(value.to_vec()).ok(), + Value::Integer(value) => Some(value.to_string()), + _ => None, + }) +} + +fn field_bytes(fields: &HashMap, key: &str) -> Option> { + fields.get(key).and_then(|value| match value { + Value::Bytes(value) => Some(value.to_vec()), + Value::String(value) => Some(value.as_bytes().to_vec()), + _ => None, + }) +} + +fn field_u64(fields: &HashMap, key: &str) -> Option { + fields.get(key).and_then(|value| match value { + Value::Integer(value) => (*value).try_into().ok(), + Value::String(value) => value.parse().ok(), + Value::Bytes(value) => std::str::from_utf8(value).ok().and_then(|value| value.parse().ok()), + _ => None, + }) +} + +fn field_u32(fields: &HashMap, key: &str) -> Option { + field_u64(fields, key).and_then(|value| value.try_into().ok()) +} diff --git a/binary/ai-gateway-service/src/main.rs b/binary/ai-gateway-service/src/main.rs index cf3ad9bb..7fa02aa1 100644 --- a/binary/ai-gateway-service/src/main.rs +++ b/binary/ai-gateway-service/src/main.rs @@ -1,1642 +1,6 @@ -use std::collections::HashMap; -use std::net::{IpAddr, SocketAddr}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; - -use axum::body::Body; -use axum::extract::{DefaultBodyLimit, Path, State}; -use axum::http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode, Uri}; -use axum::response::{IntoResponse, Response}; -use axum::routing::{get, post}; -use axum::{Json, Router}; -use base64::Engine; -use clap::Parser; -use fred::clients::{Client as FredClient, SubscriberClient}; -use fred::prelude::*; -use fred::types::streams::XReadResponse; -use fred::types::ExpireOptions; -use futures_util::StreamExt; -use serde::{Deserialize, Serialize}; -use tokio::sync::Semaphore; -use tower_http::trace::TraceLayer; - -static JOB_COUNTER: AtomicU64 = AtomicU64::new(1); - -const TOKEN_BUCKET_LUA: &str = r#" -local tokens_key = KEYS[1] -local ts_key = KEYS[2] -local rate = tonumber(ARGV[1]) -local burst = tonumber(ARGV[2]) -local now = tonumber(ARGV[3]) -local cost = tonumber(ARGV[4]) - -if rate <= 0 or burst <= 0 or cost <= 0 then - return {0, 0, 1000} -end - -local burst_milli = burst * 1000 -local cost_milli = cost * 1000 -local tokens = tonumber(redis.call('GET', tokens_key) or burst_milli) -local last_ts = tonumber(redis.call('GET', ts_key) or now) -local elapsed = math.max(0, now - last_ts) -tokens = math.min(burst_milli, tokens + elapsed * rate) - -local ttl = math.max(1000, math.ceil((burst_milli / rate) * 2)) -if tokens >= cost_milli then - tokens = tokens - cost_milli - redis.call('SET', tokens_key, tokens, 'PX', ttl) - redis.call('SET', ts_key, now, 'PX', ttl) - return {1, tokens, 0} -else - local wait_ms = math.ceil((cost_milli - tokens) / rate) - redis.call('SET', tokens_key, tokens, 'PX', ttl) - redis.call('SET', ts_key, now, 'PX', ttl) - return {0, tokens, wait_ms} -end -"#; - -#[derive(Debug, Clone, Parser)] -#[command(version, about = "External Redis-backed rate-limit and queue service for SpaceGate AI gateway")] -struct Args { - #[arg(long, env = "AI_GATEWAY_SERVICE_HOST", default_value = "0.0.0.0")] - host: IpAddr, - #[arg(long, env = "AI_GATEWAY_SERVICE_PORT", default_value_t = 18080)] - port: u16, - #[arg(long, env = "REDIS_URL", default_value = "redis://127.0.0.1/")] - redis_url: String, - #[arg(long, env = "AI_QUEUE_STREAM", default_value = "ai:jobs")] - stream_key: String, - #[arg(long, env = "AI_QUEUE_HIGH_STREAM", default_value = "ai:jobs:high")] - high_priority_stream_key: String, - #[arg(long, env = "AI_QUEUE_LOW_STREAM", default_value = "ai:jobs:low")] - low_priority_stream_key: String, - #[arg(long, env = "AI_ENABLE_PRIORITY_STREAMS", default_value_t = false)] - enable_priority_streams: bool, - #[arg(long, env = "AI_QUEUE_MAX_LEN", default_value_t = 100_000)] - stream_max_len: u64, - #[arg(long, env = "AI_QUEUE_GROUP", default_value = "ai-gateway-workers")] - consumer_group: String, - #[arg(long, env = "AI_QUEUE_CONSUMER", default_value = "ai-gateway-service")] - consumer_name: String, - #[arg(long, env = "AI_JOB_DLQ_STREAM", default_value = "ai:job-dlq")] - job_dlq_stream: String, - #[arg(long, env = "AI_CALLBACK_RETRY_STREAM", default_value = "ai:callback-retry")] - callback_retry_stream: String, - #[arg(long, env = "AI_CALLBACK_RETRY_GROUP", default_value = "ai-gateway-callbacks")] - callback_retry_group: String, - #[arg(long, env = "AI_CALLBACK_DLQ_STREAM", default_value = "ai:callback-dlq")] - callback_dlq_stream: String, - #[arg(long, env = "AI_CALLBACK_MAX_RETRY_ATTEMPTS", default_value_t = 5)] - callback_max_retry_attempts: u32, - #[arg(long, env = "AI_CALLBACK_RETRY_INITIAL_DELAY_MS", default_value_t = 1000)] - callback_retry_initial_delay_ms: u64, - #[arg(long, env = "AI_CALLBACK_RETRY_MAX_DELAY_MS", default_value_t = 60_000)] - callback_retry_max_delay_ms: u64, - #[arg(long, env = "AI_CALLBACK_RETRY_RECLAIM_IDLE_SECS", default_value_t = 60)] - callback_retry_reclaim_idle_secs: u64, - #[arg(long, env = "AI_RESULT_KEY_PREFIX", default_value = "result:")] - result_key_prefix: String, - #[arg(long, env = "AI_RESULT_CHANNEL_PREFIX", default_value = "result:")] - result_channel_prefix: String, - #[arg(long, env = "AI_RESULT_TTL_SECS", default_value_t = 120)] - result_ttl_secs: u64, - #[arg(long, env = "AI_RATE_LIMIT_RPS", default_value_t = 100)] - rate_limit_rps: u64, - #[arg(long, env = "AI_RATE_LIMIT_BURST", default_value_t = 200)] - rate_limit_burst: u64, - #[arg(long, env = "AI_TENANT_RATE_LIMIT_PREFIX", default_value = "ai:tenant:ratelimit:")] - tenant_rate_limit_prefix: String, - #[arg(long, env = "AI_WAIT_TIMEOUT_SECS", default_value_t = 60)] - wait_timeout_secs: u64, - #[arg(long, env = "AI_WORKER_CONCURRENCY", default_value_t = 1)] - worker_concurrency: usize, - #[arg(long, env = "AI_UPSTREAM_BASE_URL")] - upstream_base_url: Option, - #[arg(long, env = "AI_MAX_BODY_BYTES", default_value_t = 32 * 1024 * 1024)] - max_body_bytes: usize, - #[arg(long, env = "AI_INLINE_THRESHOLD", default_value_t = 128 * 1024)] - inline_threshold: usize, - #[arg(long, env = "AI_BODY_READ_CONCURRENCY", default_value_t = 200)] - body_read_concurrency: usize, - #[arg(long, env = "AI_RECLAIM_INTERVAL_SECS", default_value_t = 30)] - reclaim_interval_secs: u64, - #[arg(long, env = "AI_RECLAIM_MIN_IDLE_SECS", default_value_t = 30)] - reclaim_min_idle_secs: u64, - #[arg(long, env = "AI_JOB_PROCESS_LEASE_SECS", default_value_t = 120)] - job_process_lease_secs: u64, - #[arg(long, env = "AI_JOB_MAX_DELIVERY_ATTEMPTS", default_value_t = 5)] - job_max_delivery_attempts: u32, - #[arg(long, env = "AI_REQUIRE_HTTPS_CALLBACK", default_value_t = true)] - require_https_callback: bool, - #[arg(long, env = "AI_OBJECT_STORE_ENDPOINT")] - object_store_endpoint: Option, - #[arg(long, env = "AI_OBJECT_STORE_BUCKET", default_value = "ai-gateway-body")] - object_store_bucket: String, - #[arg(long, env = "AI_OBJECT_STORE_PREFIX", default_value = "bodies")] - object_store_prefix: String, - #[arg(long, env = "AI_OBJECT_MULTIPART_PART_SIZE", default_value_t = 5 * 1024 * 1024)] - object_multipart_part_size: usize, - #[arg(long, env = "AI_OBJECT_STORE_AUTH_HEADER")] - object_store_auth_header: Option, -} - -#[derive(Clone)] -struct AppState { - redis: FredClient, - http: reqwest::Client, - cfg: Arc, - body_permits: Arc, - metrics: Arc, -} - -#[derive(Default)] -struct Metrics { - rate_limited_total: AtomicU64, - enqueue_total: AtomicU64, - enqueue_queue_total: AtomicU64, - enqueue_wait_total: AtomicU64, - enqueue_latency_count: AtomicU64, - enqueue_latency_sum_ms: AtomicU64, - enqueue_latency_le_100_ms: AtomicU64, - enqueue_latency_le_500_ms: AtomicU64, - enqueue_latency_le_1000_ms: AtomicU64, - enqueue_latency_gt_1000_ms: AtomicU64, - body_size_le_10kb: AtomicU64, - body_size_le_128kb: AtomicU64, - body_size_le_5mb: AtomicU64, - body_size_gt_5mb: AtomicU64, - body_size_count: AtomicU64, - body_size_sum_bytes: AtomicU64, - wait_total: AtomicU64, - wait_timeout_total: AtomicU64, - callback_failure_total: AtomicU64, - callback_retry_total: AtomicU64, - callback_retry_success_total: AtomicU64, - callback_retry_dlq_total: AtomicU64, - worker_completed_total: AtomicU64, - worker_failed_total: AtomicU64, - worker_processing_count: AtomicU64, - worker_processing_sum_ms: AtomicU64, - worker_processing_le_1000_ms: AtomicU64, - worker_processing_le_5000_ms: AtomicU64, - worker_processing_le_30000_ms: AtomicU64, - worker_processing_gt_30000_ms: AtomicU64, - reclaimed_total: AtomicU64, - job_dlq_total: AtomicU64, - lease_skip_total: AtomicU64, - object_offload_total: AtomicU64, - object_multipart_abort_total: AtomicU64, -} - -#[derive(Debug, Serialize)] -struct RateLimitResponse { - allowed: bool, - remaining_tokens_milli: i64, - retry_after_ms: i64, -} - -#[derive(Debug, Serialize)] -struct EnqueueResponse { - job_id: String, - stream_id: String, - stream_key: String, - status: &'static str, - poll_url: String, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -struct StoredResult { - job_id: String, - status: String, - http_status: u16, - headers: HashMap, - body_base64: String, - completed_at_ms: u64, - error: Option, -} - -#[derive(Debug)] -struct AcceptedJob { - response: EnqueueResponse, - created_at_ms: u64, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum QueuePolicy { - Queue, - Wait, -} - -impl QueuePolicy { - fn as_str(self) -> &'static str { - match self { - QueuePolicy::Queue => "queue", - QueuePolicy::Wait => "wait", - } - } -} - -#[derive(Debug)] -struct BodyLocation { - body_base64: String, - object_ref: String, - size: usize, - storage: &'static str, -} - -#[derive(Debug)] -struct CompletedPart { - part_number: usize, - etag: String, -} - -#[derive(Debug)] -struct ServiceError { - status: StatusCode, - message: String, -} - -impl ServiceError { - fn bad_request(message: impl Into) -> Self { - Self { - status: StatusCode::BAD_REQUEST, - message: message.into(), - } - } - - fn internal(message: impl Into) -> Self { - Self { - status: StatusCode::INTERNAL_SERVER_ERROR, - message: message.into(), - } - } - - fn gateway_timeout(message: impl Into) -> Self { - Self { - status: StatusCode::GATEWAY_TIMEOUT, - message: message.into(), - } - } - - fn payload_too_large(message: impl Into) -> Self { - Self { - status: StatusCode::PAYLOAD_TOO_LARGE, - message: message.into(), - } - } -} - -impl IntoResponse for ServiceError { - fn into_response(self) -> Response { - let body = Json(serde_json::json!({ "error": self.message })); - (self.status, body).into_response() - } -} - -impl std::fmt::Display for ServiceError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.message) - } -} - -impl std::error::Error for ServiceError {} - -impl From for ServiceError { - fn from(value: fred::error::Error) -> Self { - Self::internal(format!("redis: {value}")) - } -} - -impl From for ServiceError { - fn from(value: reqwest::Error) -> Self { - Self::internal(format!("http: {value}")) - } -} +mod app; #[tokio::main] async fn main() -> Result<(), Box> { - tracing_subscriber::fmt().with_env_filter(tracing_subscriber::EnvFilter::from_default_env()).init(); - - let args = Args::parse(); - let redis = build_redis_client(&args.redis_url)?; - let _redis_task = redis.init().await?; - let state = AppState { - redis, - http: reqwest::Client::new(), - cfg: Arc::new(args.clone()), - body_permits: Arc::new(Semaphore::new(args.body_read_concurrency.max(1))), - metrics: Arc::new(Metrics::default()), - }; - - ensure_consumer_groups(&state).await?; - if state.cfg.upstream_base_url.is_some() { - spawn_workers(state.clone()); - spawn_reclaimer(state.clone()); - spawn_callback_retry_worker(state.clone()); - } else { - tracing::warn!("AI_UPSTREAM_BASE_URL is not set; queue jobs will be stored but no local worker will process them"); - } - - let app = Router::new() - .route("/healthz", get(healthz)) - .route("/metrics", get(metrics)) - .route("/v1/ratelimit/check", post(check_rate_limit)) - .route("/v1/queue/enqueue", post(enqueue)) - .route("/v1/queue/enqueue-and-wait", post(enqueue_and_wait)) - .route("/v1/jobs/{job_id}", get(get_job)) - .layer(DefaultBodyLimit::max(args.max_body_bytes)) - .layer(TraceLayer::new_for_http()) - .with_state(state); - - let addr = SocketAddr::new(args.host, args.port); - tracing::info!(%addr, "ai-gateway-service listening"); - let listener = tokio::net::TcpListener::bind(addr).await?; - axum::serve(listener, app).await?; - Ok(()) -} - -async fn healthz() -> &'static str { - "ok" -} - -async fn check_rate_limit(State(state): State, headers: HeaderMap, uri: Uri) -> Result, ServiceError> { - let tenant = required_header(&headers, "x-tenant-id")?; - let model = optional_header(&headers, "x-model").unwrap_or_else(|| "default".to_string()); - let path = optional_header(&headers, "x-original-path").unwrap_or_else(|| uri.path().to_string()); - let (rate_limit_rps, rate_limit_burst) = tenant_rate_limit(&state, &tenant).await?; - let key = sanitize_key(&format!("{tenant}:{model}:{path}")); - let tokens_key = format!("ai:ratelimit:{key}:tokens"); - let ts_key = format!("ai:ratelimit:{key}:ts"); - let now = now_ms(); - - let out: Vec = state - .redis - .eval( - TOKEN_BUCKET_LUA, - vec![tokens_key, ts_key], - vec![rate_limit_rps.to_string(), rate_limit_burst.to_string(), now.to_string(), "1".to_string()], - ) - .await?; - - let allowed = out.first().copied().unwrap_or(0) == 1; - if !allowed { - state.metrics.rate_limited_total.fetch_add(1, Ordering::Relaxed); - } - Ok(Json(RateLimitResponse { - allowed, - remaining_tokens_milli: out.get(1).copied().unwrap_or(0), - retry_after_ms: out.get(2).copied().unwrap_or(0), - })) -} - -async fn enqueue(State(state): State, method: Method, uri: Uri, headers: HeaderMap, body: Body) -> Result { - let accepted = enqueue_job(&state, QueuePolicy::Queue, method, uri, headers, body).await?; - let mut resp = (StatusCode::ACCEPTED, Json(&accepted.response)).into_response(); - resp.headers_mut().insert("x-job-id", header_value(&accepted.response.job_id)?); - resp.headers_mut().insert("location", header_value(&accepted.response.poll_url)?); - Ok(resp) -} - -async fn enqueue_and_wait(State(state): State, method: Method, uri: Uri, headers: HeaderMap, body: Body) -> Result { - let timeout_secs = optional_header(&headers, "x-request-timeout").and_then(|v| v.parse::().ok()).unwrap_or(state.cfg.wait_timeout_secs); - state.metrics.wait_total.fetch_add(1, Ordering::Relaxed); - let accepted = enqueue_job(&state, QueuePolicy::Wait, method, uri, headers, body).await?; - let channel = result_channel(&state, &accepted.response.job_id); - let subscriber = build_subscriber_client(&state.cfg.redis_url)?; - let _subscriber_task = subscriber.init().await?; - subscriber.subscribe(channel.as_str()).await?; - - if let Some(result) = load_result(&state, &accepted.response.job_id).await? { - let _ = subscriber.quit().await; - return Ok(result_to_response(result, accepted.created_at_ms)?); - } - - let mut messages = subscriber.message_rx(); - let wait = tokio::time::timeout(Duration::from_secs(timeout_secs), async { - loop { - let message = messages.recv().await.map_err(|e| ServiceError::internal(format!("pubsub receive: {e}")))?; - if &*message.channel == channel.as_str() { - return Ok::<(), ServiceError>(()); - } - } - }) - .await; - match wait { - Ok(Ok(())) => { - let _ = subscriber.quit().await; - if let Some(result) = load_result(&state, &accepted.response.job_id).await? { - Ok(result_to_response(result, accepted.created_at_ms)?) - } else { - Err(ServiceError::gateway_timeout(format!( - "job {} completed notification received but result is missing", - accepted.response.job_id - ))) - } - } - _ => { - let _ = subscriber.quit().await; - state.metrics.wait_timeout_total.fetch_add(1, Ordering::Relaxed); - let waited_ms = now_ms().saturating_sub(accepted.created_at_ms); - let body = Json(serde_json::json!({ - "error": "timeout", - "job_id": accepted.response.job_id, - "poll_url": accepted.response.poll_url, - "waited_ms": waited_ms, - "message": "Job is still processing. Switch to queue mode with a callback for long tasks." - })); - Ok((StatusCode::GATEWAY_TIMEOUT, body).into_response()) - } - } -} - -async fn get_job(State(state): State, Path(job_id): Path) -> Result { - match load_result(&state, &job_id).await? { - Some(result) => Ok(Json(result).into_response()), - None => Ok((StatusCode::NOT_FOUND, Json(serde_json::json!({ "error": "not_found", "job_id": job_id }))).into_response()), - } -} - -async fn metrics(State(state): State) -> Result { - let queue_depth: i64 = state.redis.xlen(state.cfg.stream_key.as_str()).await.unwrap_or_default(); - let high_queue_depth: i64 = state.redis.xlen(state.cfg.high_priority_stream_key.as_str()).await.unwrap_or_default(); - let low_queue_depth: i64 = state.redis.xlen(state.cfg.low_priority_stream_key.as_str()).await.unwrap_or_default(); - let job_dlq_depth: i64 = state.redis.xlen(state.cfg.job_dlq_stream.as_str()).await.unwrap_or_default(); - let callback_retry_depth: i64 = state.redis.xlen(state.cfg.callback_retry_stream.as_str()).await.unwrap_or_default(); - let callback_dlq_depth: i64 = state.redis.xlen(state.cfg.callback_dlq_stream.as_str()).await.unwrap_or_default(); - let pel_size = pending_size(&state, &state.cfg.stream_key).await; - let high_pel_size = pending_size(&state, &state.cfg.high_priority_stream_key).await; - let low_pel_size = pending_size(&state, &state.cfg.low_priority_stream_key).await; - let callback_retry_pel_size = pending_size_for_group(&state, &state.cfg.callback_retry_stream, &state.cfg.callback_retry_group).await; - - let body = format!( - "\ -rate_limited_total {}\n\ -enqueue_total {}\n\ -enqueue_total{{policy=\"queue\"}} {}\n\ -enqueue_total{{policy=\"wait\"}} {}\n\ -enqueue_latency_ms_count {}\n\ -enqueue_latency_ms_sum {}\n\ -enqueue_latency_ms_bucket{{le=\"100\"}} {}\n\ -enqueue_latency_ms_bucket{{le=\"500\"}} {}\n\ -enqueue_latency_ms_bucket{{le=\"1000\"}} {}\n\ -enqueue_latency_ms_bucket{{le=\"+Inf\"}} {}\n\ -enqueue_body_size_bytes_count {}\n\ -enqueue_body_size_bytes_sum {}\n\ -enqueue_body_size_bytes_bucket{{le=\"10240\"}} {}\n\ -enqueue_body_size_bytes_bucket{{le=\"131072\"}} {}\n\ -enqueue_body_size_bytes_bucket{{le=\"5242880\"}} {}\n\ -enqueue_body_size_bytes_bucket{{le=\"+Inf\"}} {}\n\ -wait_total {}\n\ -wait_timeout_total {}\n\ -callback_failure_total {}\n\ -callback_retry_total {}\n\ -callback_retry_success_total {}\n\ -callback_retry_dlq_total {}\n\ -worker_completed_total {}\n\ -worker_failed_total {}\n\ -worker_processing_time_ms_count {}\n\ -worker_processing_time_ms_sum {}\n\ -worker_processing_time_ms_bucket{{le=\"1000\"}} {}\n\ -worker_processing_time_ms_bucket{{le=\"5000\"}} {}\n\ -worker_processing_time_ms_bucket{{le=\"30000\"}} {}\n\ -worker_processing_time_ms_bucket{{le=\"+Inf\"}} {}\n\ -reclaimed_total {}\n\ -job_dlq_total {}\n\ -lease_skip_total {}\n\ -object_offload_total {}\n\ -object_multipart_abort_total {}\n\ -queue_depth {}\n\ -queue_depth{{priority=\"high\"}} {}\n\ -queue_depth{{priority=\"low\"}} {}\n\ -pel_size {}\n\ -pel_size{{priority=\"high\"}} {}\n\ -pel_size{{priority=\"low\"}} {}\n\ -job_dlq_depth {}\n\ -callback_retry_depth {}\n\ -callback_retry_pel_size {}\n\ -callback_dlq_depth {}\n", - state.metrics.rate_limited_total.load(Ordering::Relaxed), - state.metrics.enqueue_total.load(Ordering::Relaxed), - state.metrics.enqueue_queue_total.load(Ordering::Relaxed), - state.metrics.enqueue_wait_total.load(Ordering::Relaxed), - state.metrics.enqueue_latency_count.load(Ordering::Relaxed), - state.metrics.enqueue_latency_sum_ms.load(Ordering::Relaxed), - state.metrics.enqueue_latency_le_100_ms.load(Ordering::Relaxed), - state.metrics.enqueue_latency_le_100_ms.load(Ordering::Relaxed) + state.metrics.enqueue_latency_le_500_ms.load(Ordering::Relaxed), - state.metrics.enqueue_latency_le_100_ms.load(Ordering::Relaxed) - + state.metrics.enqueue_latency_le_500_ms.load(Ordering::Relaxed) - + state.metrics.enqueue_latency_le_1000_ms.load(Ordering::Relaxed), - state.metrics.enqueue_latency_count.load(Ordering::Relaxed), - state.metrics.body_size_count.load(Ordering::Relaxed), - state.metrics.body_size_sum_bytes.load(Ordering::Relaxed), - state.metrics.body_size_le_10kb.load(Ordering::Relaxed), - state.metrics.body_size_le_10kb.load(Ordering::Relaxed) + state.metrics.body_size_le_128kb.load(Ordering::Relaxed), - state.metrics.body_size_le_10kb.load(Ordering::Relaxed) + state.metrics.body_size_le_128kb.load(Ordering::Relaxed) + state.metrics.body_size_le_5mb.load(Ordering::Relaxed), - state.metrics.body_size_count.load(Ordering::Relaxed), - state.metrics.wait_total.load(Ordering::Relaxed), - state.metrics.wait_timeout_total.load(Ordering::Relaxed), - state.metrics.callback_failure_total.load(Ordering::Relaxed), - state.metrics.callback_retry_total.load(Ordering::Relaxed), - state.metrics.callback_retry_success_total.load(Ordering::Relaxed), - state.metrics.callback_retry_dlq_total.load(Ordering::Relaxed), - state.metrics.worker_completed_total.load(Ordering::Relaxed), - state.metrics.worker_failed_total.load(Ordering::Relaxed), - state.metrics.worker_processing_count.load(Ordering::Relaxed), - state.metrics.worker_processing_sum_ms.load(Ordering::Relaxed), - state.metrics.worker_processing_le_1000_ms.load(Ordering::Relaxed), - state.metrics.worker_processing_le_1000_ms.load(Ordering::Relaxed) + state.metrics.worker_processing_le_5000_ms.load(Ordering::Relaxed), - state.metrics.worker_processing_le_1000_ms.load(Ordering::Relaxed) - + state.metrics.worker_processing_le_5000_ms.load(Ordering::Relaxed) - + state.metrics.worker_processing_le_30000_ms.load(Ordering::Relaxed), - state.metrics.worker_processing_count.load(Ordering::Relaxed), - state.metrics.reclaimed_total.load(Ordering::Relaxed), - state.metrics.job_dlq_total.load(Ordering::Relaxed), - state.metrics.lease_skip_total.load(Ordering::Relaxed), - state.metrics.object_offload_total.load(Ordering::Relaxed), - state.metrics.object_multipart_abort_total.load(Ordering::Relaxed), - queue_depth, - high_queue_depth, - low_queue_depth, - pel_size, - high_pel_size, - low_pel_size, - job_dlq_depth, - callback_retry_depth, - callback_retry_pel_size, - callback_dlq_depth, - ); - Ok((StatusCode::OK, [("content-type", "text/plain; version=0.0.4")], body).into_response()) -} - -async fn enqueue_job(state: &AppState, policy: QueuePolicy, _method: Method, uri: Uri, headers: HeaderMap, body: Body) -> Result { - let enqueue_started_at = now_ms(); - let _permit = state.body_permits.acquire().await.map_err(|_| ServiceError::internal("body semaphore closed"))?; - let job_id = new_job_id(); - let tenant_id = required_header(&headers, "x-tenant-id")?; - let model = optional_header(&headers, "x-model").unwrap_or_else(|| "default".to_string()); - let callback_url = optional_header(&headers, "x-callback-url").unwrap_or_default(); - validate_callback_url(state, policy, &callback_url)?; - let original_method = optional_header(&headers, "x-original-method").unwrap_or_else(|| "POST".to_string()); - let original_path = optional_header(&headers, "x-original-path").unwrap_or_else(|| uri.path().to_string()); - let request_headers = headers_to_json(&headers)?; - let created_at = now_ms(); - let body_ref = store_body(state, &job_id, body).await?; - let stream_key = stream_for_request(state, &headers); - - let stream_id: String = state - .redis - .xadd( - stream_key.as_str(), - false, - None::<()>, - "*", - vec![ - ("job_id", Value::String(job_id.clone().into())), - ("tenant_id", Value::String(tenant_id.into())), - ("policy", Value::String(policy.as_str().into())), - ("model", Value::String(model.into())), - ("method", Value::String(original_method.into())), - ("path", Value::String(original_path.into())), - ("headers", Value::String(request_headers.into())), - ("body", Value::String(body_ref.body_base64.into())), - ("ref", Value::String(body_ref.object_ref.into())), - ("size", Value::Integer(body_ref.size as i64)), - ("storage", Value::String(body_ref.storage.into())), - ("callback_url", Value::String(callback_url.into())), - ("created_at", Value::Integer(created_at as i64)), - ], - ) - .await?; - trim_stream(state, &stream_key).await?; - - state.metrics.enqueue_total.fetch_add(1, Ordering::Relaxed); - observe_enqueue_latency(&state.metrics, now_ms().saturating_sub(enqueue_started_at)); - observe_body_size(&state.metrics, body_ref.size); - match policy { - QueuePolicy::Queue => { - state.metrics.enqueue_queue_total.fetch_add(1, Ordering::Relaxed); - } - QueuePolicy::Wait => { - state.metrics.enqueue_wait_total.fetch_add(1, Ordering::Relaxed); - } - } - - Ok(AcceptedJob { - response: EnqueueResponse { - job_id: job_id.clone(), - stream_id, - stream_key, - status: "queued", - poll_url: format!("/v1/jobs/{job_id}"), - }, - created_at_ms: created_at, - }) -} - -fn spawn_workers(state: AppState) { - for idx in 0..state.cfg.worker_concurrency.max(1) { - let state = state.clone(); - tokio::spawn(async move { - let consumer = format!("{}-{idx}", state.cfg.consumer_name); - loop { - if let Err(e) = worker_once(&state, &consumer).await { - tracing::warn!(error = %e.message, "worker loop failed"); - tokio::time::sleep(Duration::from_secs(1)).await; - } - } - }); - } -} - -async fn worker_once(state: &AppState, consumer: &str) -> Result<(), ServiceError> { - let streams = worker_streams(state); - for (idx, stream) in streams.iter().enumerate() { - let block = if idx + 1 == streams.len() { 1000 } else { 10 }; - let processed = read_worker_stream(state, consumer, stream, block).await?; - if processed > 0 { - return Ok(()); - } - } - Ok(()) -} - -async fn read_worker_stream(state: &AppState, consumer: &str, stream: &str, block_ms: u64) -> Result { - let reply: XReadResponse = - state.redis.xreadgroup_map(state.cfg.consumer_group.as_str(), consumer, Some(5), Some(block_ms), false, vec![stream], vec![">"]).await?; - - let mut processed = 0; - for (_stream, entries) in reply { - for (entry_id, fields) in entries { - match process_stream_entry(state, stream, entry_id.as_str(), &fields).await { - Ok(true) => { - processed += 1; - } - Ok(false) => {} - Err(e) => { - tracing::warn!(stream_id = %entry_id, error = %e.message, "job processing failed"); - state.metrics.worker_failed_total.fetch_add(1, Ordering::Relaxed); - } - } - } - } - Ok(processed) -} - -async fn process_stream_entry(state: &AppState, stream: &str, entry_id: &str, fields: &HashMap) -> Result { - let job_id = field_string(fields, "job_id").ok_or_else(|| ServiceError::bad_request("job missing job_id"))?; - let lease_owner = format!("{}:{stream}:{entry_id}:{}", state.cfg.consumer_name, now_ms()); - - if !acquire_job_lease(state, &job_id, &lease_owner).await? { - state.metrics.lease_skip_total.fetch_add(1, Ordering::Relaxed); - tracing::info!(job_id = %job_id, stream = %stream, entry_id = %entry_id, "job is already leased; skip reclaimed duplicate"); - return Ok(false); - } - - let attempt = increment_job_delivery_attempt(state, &job_id).await?; - if attempt > state.cfg.job_max_delivery_attempts { - enqueue_job_dlq(state, stream, entry_id, fields, attempt, "max_delivery_attempts_exceeded").await?; - ack_stream_entry(state, stream, entry_id).await?; - release_job_lease(state, &job_id).await; - state.metrics.job_dlq_total.fetch_add(1, Ordering::Relaxed); - return Ok(true); - } - - let processing_started_at = now_ms(); - match process_job(state, stream, entry_id, fields).await { - Ok(()) => { - observe_worker_processing(&state.metrics, now_ms().saturating_sub(processing_started_at)); - ack_stream_entry(state, stream, entry_id).await?; - clear_job_delivery_attempt(state, &job_id).await; - release_job_lease(state, &job_id).await; - Ok(true) - } - Err(e) => { - observe_worker_processing(&state.metrics, now_ms().saturating_sub(processing_started_at)); - release_job_lease(state, &job_id).await; - Err(e) - } - } -} - -async fn process_job(state: &AppState, _stream: &str, _stream_id: &str, fields: &HashMap) -> Result<(), ServiceError> { - let Some(base) = state.cfg.upstream_base_url.as_deref() else { - return Err(ServiceError::internal("upstream base URL is not configured")); - }; - let job_id = field_string(fields, "job_id").ok_or_else(|| ServiceError::bad_request("job missing job_id"))?; - let method = field_string(fields, "method").unwrap_or_else(|| "POST".to_string()); - let path = field_string(fields, "path").unwrap_or_else(|| "/".to_string()); - let headers_json = field_string(fields, "headers").unwrap_or_else(|| "{}".to_string()); - let callback_url = field_string(fields, "callback_url").unwrap_or_default(); - let body = load_body(state, fields).await?; - let headers: HashMap = serde_json::from_str(&headers_json).unwrap_or_default(); - - let url = format!("{}{}", base.trim_end_matches('/'), path); - let parsed_method = method.parse::().unwrap_or(reqwest::Method::POST); - let mut req = state.http.request(parsed_method, url); - for (name, value) in headers { - if should_forward_header(&name) { - req = req.header(name, value); - } - } - let upstream = req.body(body).send().await; - let result = match upstream { - Ok(resp) => { - let status = resp.status().as_u16(); - let mut headers = HashMap::new(); - for (name, value) in resp.headers() { - if let Ok(value) = value.to_str() { - headers.insert(name.as_str().to_string(), value.to_string()); - } - } - let body = resp.bytes().await.unwrap_or_default(); - StoredResult { - job_id: job_id.clone(), - status: "completed".to_string(), - http_status: status, - headers, - body_base64: base64::engine::general_purpose::STANDARD.encode(body), - completed_at_ms: now_ms(), - error: None, - } - } - Err(e) => StoredResult { - job_id: job_id.clone(), - status: "failed".to_string(), - http_status: 502, - headers: HashMap::new(), - body_base64: String::new(), - completed_at_ms: now_ms(), - error: Some(e.to_string()), - }, - }; - - store_result(state, &result).await?; - if !callback_url.is_empty() { - let callback_body = callback_body(&result); - if let Err(e) = post_callback(state, &callback_url, &job_id, &callback_body).await { - tracing::warn!(job_id = %job_id, error = %e.message, "callback failed"); - state.metrics.callback_failure_total.fetch_add(1, Ordering::Relaxed); - enqueue_callback_retry(state, &callback_url, &job_id, &callback_body, e.message.as_str()).await?; - } - } - state.metrics.worker_completed_total.fetch_add(1, Ordering::Relaxed); - Ok(()) -} - -fn callback_body(result: &StoredResult) -> serde_json::Value { - serde_json::json!({ - "job_id": result.job_id, - "status": result.status, - "http_status": result.http_status, - "headers": result.headers, - "body_base64": result.body_base64, - "result": result.body_base64, - "completed_at_ms": result.completed_at_ms, - "error": result.error, - }) -} - -async fn post_callback(state: &AppState, callback_url: &str, job_id: &str, body: &serde_json::Value) -> Result<(), ServiceError> { - state.http.post(callback_url).header("x-gateway-job-id", job_id).json(body).send().await?.error_for_status()?; - Ok(()) -} - -async fn enqueue_callback_retry(state: &AppState, callback_url: &str, job_id: &str, body: &serde_json::Value, last_error: &str) -> Result<(), ServiceError> { - let body = serde_json::to_string(body).map_err(|e| ServiceError::internal(format!("serialize callback retry: {e}")))?; - enqueue_callback_retry_raw( - state, - callback_url, - job_id, - &body, - 1, - now_ms().saturating_add(state.cfg.callback_retry_initial_delay_ms), - last_error, - ) - .await -} - -async fn enqueue_callback_retry_raw( - state: &AppState, - callback_url: &str, - job_id: &str, - body: &str, - attempt: u32, - next_attempt_at_ms: u64, - last_error: &str, -) -> Result<(), ServiceError> { - let _: String = state - .redis - .xadd( - state.cfg.callback_retry_stream.as_str(), - false, - None::<()>, - "*", - vec![ - ("job_id", Value::String(job_id.to_string().into())), - ("callback_url", Value::String(callback_url.to_string().into())), - ("body", Value::String(body.to_string().into())), - ("attempt", Value::Integer(attempt as i64)), - ("next_attempt_at_ms", Value::Integer(next_attempt_at_ms as i64)), - ("last_error", Value::String(last_error.to_string().into())), - ("created_at", Value::Integer(now_ms() as i64)), - ], - ) - .await?; - trim_stream(state, &state.cfg.callback_retry_stream).await?; - state.metrics.callback_retry_total.fetch_add(1, Ordering::Relaxed); - Ok(()) -} - -fn spawn_callback_retry_worker(state: AppState) { - tokio::spawn(async move { - loop { - if let Err(e) = callback_retry_once(&state).await { - tracing::warn!(error = %e.message, "callback retry loop failed"); - tokio::time::sleep(Duration::from_secs(1)).await; - } - } - }); -} - -async fn callback_retry_once(state: &AppState) -> Result<(), ServiceError> { - reclaim_callback_retries(state).await?; - let reply: XReadResponse = state - .redis - .xreadgroup_map( - state.cfg.callback_retry_group.as_str(), - state.cfg.consumer_name.as_str(), - Some(5), - Some(1000), - false, - vec![state.cfg.callback_retry_stream.as_str()], - vec![">"], - ) - .await?; - - for (_stream, entries) in reply { - for (entry_id, fields) in entries { - process_callback_retry_entry(state, entry_id.as_str(), &fields).await?; - } - } - Ok(()) -} - -async fn reclaim_callback_retries(state: &AppState) -> Result<(), ServiceError> { - let consumer = format!("{}-callback-reclaimer", state.cfg.consumer_name); - let min_idle_ms = state.cfg.callback_retry_reclaim_idle_secs.saturating_mul(1000); - let (_cursor, entries): (String, Vec<(String, HashMap)>) = state - .redis - .xautoclaim_values( - state.cfg.callback_retry_stream.as_str(), - state.cfg.callback_retry_group.as_str(), - consumer.as_str(), - min_idle_ms, - "0-0", - Some(10), - false, - ) - .await?; - for (entry_id, fields) in entries { - process_callback_retry_entry(state, entry_id.as_str(), &fields).await?; - } - Ok(()) -} - -async fn process_callback_retry_entry(state: &AppState, entry_id: &str, fields: &HashMap) -> Result<(), ServiceError> { - let job_id = field_string(fields, "job_id").unwrap_or_default(); - let callback_url = field_string(fields, "callback_url").unwrap_or_default(); - let body = field_string(fields, "body").unwrap_or_else(|| "{}".to_string()); - let attempt = field_u32(fields, "attempt").unwrap_or(1); - let next_attempt_at_ms = field_u64(fields, "next_attempt_at_ms").unwrap_or(0); - let now = now_ms(); - - if next_attempt_at_ms > now { - let last_error = field_string(fields, "last_error").unwrap_or_default(); - enqueue_callback_retry_raw(state, &callback_url, &job_id, &body, attempt, next_attempt_at_ms, &last_error).await?; - ack_callback_retry(state, entry_id).await?; - return Ok(()); - } - - let parsed = serde_json::from_str::(&body).unwrap_or_else(|_| serde_json::json!({ "body": body })); - match post_callback(state, &callback_url, &job_id, &parsed).await { - Ok(()) => { - ack_callback_retry(state, entry_id).await?; - state.metrics.callback_retry_success_total.fetch_add(1, Ordering::Relaxed); - } - Err(e) => { - tracing::warn!(job_id = %job_id, attempt, error = %e.message, "callback retry failed"); - if attempt >= state.cfg.callback_max_retry_attempts { - enqueue_callback_dlq(state, &callback_url, &job_id, &parsed, attempt, &e.message).await?; - ack_callback_retry(state, entry_id).await?; - } else { - let next_attempt = attempt.saturating_add(1); - let delay_ms = callback_retry_delay_ms(state.cfg.callback_retry_initial_delay_ms, state.cfg.callback_retry_max_delay_ms, next_attempt); - let retry_body = serde_json::to_string(&parsed).unwrap_or_else(|_| "{}".to_string()); - enqueue_callback_retry_raw(state, &callback_url, &job_id, &retry_body, next_attempt, now.saturating_add(delay_ms), &e.message).await?; - ack_callback_retry(state, entry_id).await?; - } - } - } - Ok(()) -} - -async fn ack_callback_retry(state: &AppState, entry_id: &str) -> Result<(), ServiceError> { - let _: i64 = state.redis.xack(state.cfg.callback_retry_stream.as_str(), state.cfg.callback_retry_group.as_str(), vec![entry_id]).await?; - Ok(()) -} - -async fn enqueue_callback_dlq(state: &AppState, callback_url: &str, job_id: &str, body: &serde_json::Value, attempts: u32, final_error: &str) -> Result<(), ServiceError> { - let body = serde_json::to_string(body).map_err(|e| ServiceError::internal(format!("serialize callback dlq: {e}")))?; - let _: String = state - .redis - .xadd( - state.cfg.callback_dlq_stream.as_str(), - false, - None::<()>, - "*", - vec![ - ("job_id", Value::String(job_id.to_string().into())), - ("callback_url", Value::String(callback_url.to_string().into())), - ("body", Value::String(body.into())), - ("attempts", Value::Integer(attempts as i64)), - ("final_error", Value::String(final_error.to_string().into())), - ("failed_at", Value::Integer(now_ms() as i64)), - ], - ) - .await?; - trim_stream(state, &state.cfg.callback_dlq_stream).await?; - state.metrics.callback_retry_dlq_total.fetch_add(1, Ordering::Relaxed); - Ok(()) -} - -fn callback_retry_delay_ms(initial_delay_ms: u64, max_delay_ms: u64, attempt: u32) -> u64 { - let exponent = attempt.saturating_sub(1).min(16); - let multiplier = 1u64.checked_shl(exponent).unwrap_or(u64::MAX); - initial_delay_ms.saturating_mul(multiplier).min(max_delay_ms) -} - -async fn acquire_job_lease(state: &AppState, job_id: &str, owner: &str) -> Result { - let key = job_lease_key(job_id); - let result: Option = state - .redis - .set( - key, - owner, - Some(Expiration::EX(state.cfg.job_process_lease_secs.max(1) as i64)), - Some(SetOptions::NX), - false, - ) - .await?; - Ok(result.is_some()) -} - -async fn release_job_lease(state: &AppState, job_id: &str) { - let _: Result = state.redis.del(job_lease_key(job_id)).await; -} - -async fn increment_job_delivery_attempt(state: &AppState, job_id: &str) -> Result { - let key = job_attempt_key(job_id); - let attempt: i64 = state.redis.incr_by(key.as_str(), 1).await?; - let _: () = state.redis.expire(key.as_str(), state.cfg.result_ttl_secs.max(300) as i64, None::).await?; - Ok(attempt.max(0) as u32) -} - -async fn clear_job_delivery_attempt(state: &AppState, job_id: &str) { - let _: Result = state.redis.del(job_attempt_key(job_id)).await; -} - -async fn ack_stream_entry(state: &AppState, stream: &str, entry_id: &str) -> Result<(), ServiceError> { - let _: i64 = state.redis.xack(stream, state.cfg.consumer_group.as_str(), vec![entry_id]).await?; - Ok(()) -} - -async fn enqueue_job_dlq(state: &AppState, stream: &str, entry_id: &str, fields: &HashMap, attempts: u32, reason: &str) -> Result<(), ServiceError> { - let job_id = field_string(fields, "job_id").unwrap_or_default(); - let fields_json = stream_fields_to_json(fields)?; - let _: String = state - .redis - .xadd( - state.cfg.job_dlq_stream.as_str(), - false, - None::<()>, - "*", - vec![ - ("job_id", Value::String(job_id.into())), - ("source_stream", Value::String(stream.to_string().into())), - ("source_entry_id", Value::String(entry_id.to_string().into())), - ("attempts", Value::Integer(attempts as i64)), - ("reason", Value::String(reason.to_string().into())), - ("fields", Value::String(fields_json.into())), - ("failed_at", Value::Integer(now_ms() as i64)), - ], - ) - .await?; - trim_stream(state, &state.cfg.job_dlq_stream).await?; - Ok(()) -} - -fn stream_fields_to_json(fields: &HashMap) -> Result { - let mut out = HashMap::new(); - for (key, value) in fields { - if let Some(value) = field_string(fields, key) { - out.insert(key.clone(), value); - } else { - out.insert(key.clone(), format!("{value:?}")); - } - } - serde_json::to_string(&out).map_err(|e| ServiceError::internal(format!("serialize job dlq fields: {e}"))) -} - -fn job_lease_key(job_id: &str) -> String { - format!("ai:job:lease:{}", sanitize_key(job_id)) -} - -fn job_attempt_key(job_id: &str) -> String { - format!("ai:job:attempt:{}", sanitize_key(job_id)) -} - -fn spawn_reclaimer(state: AppState) { - tokio::spawn(async move { - let mut interval = tokio::time::interval(Duration::from_secs(state.cfg.reclaim_interval_secs.max(1))); - loop { - interval.tick().await; - if let Err(e) = reclaim_once(&state).await { - tracing::warn!(error = %e.message, "stream reclaim failed"); - } - } - }); -} - -async fn reclaim_once(state: &AppState) -> Result<(), ServiceError> { - let consumer = format!("{}-reclaimer", state.cfg.consumer_name); - let min_idle_ms = state.cfg.reclaim_min_idle_secs.saturating_mul(1000); - for stream in worker_streams(state) { - let (_cursor, entries): (String, Vec<(String, HashMap)>) = - state.redis.xautoclaim_values(stream.as_str(), state.cfg.consumer_group.as_str(), consumer.as_str(), min_idle_ms, "0-0", Some(10), false).await?; - for (entry_id, fields) in entries { - match process_stream_entry(state, stream.as_str(), entry_id.as_str(), &fields).await { - Ok(true) => { - state.metrics.reclaimed_total.fetch_add(1, Ordering::Relaxed); - } - Ok(false) => {} - Err(e) => { - tracing::warn!(stream = %stream, entry_id = %entry_id, error = %e.message, "reclaimed job failed"); - } - } - } - } - Ok(()) -} - -async fn ensure_consumer_groups(state: &AppState) -> Result<(), ServiceError> { - for stream in worker_streams(state) { - ensure_consumer_group(state, &stream, &state.cfg.consumer_group).await?; - } - ensure_consumer_group(state, &state.cfg.callback_retry_stream, &state.cfg.callback_retry_group).await?; - Ok(()) -} - -async fn ensure_consumer_group(state: &AppState, stream: &str, group: &str) -> Result<(), ServiceError> { - let res: FredResult = state.redis.xgroup_create(stream, group, "$", true).await; - match res { - Ok(_) => Ok(()), - Err(e) if e.to_string().contains("BUSYGROUP") => Ok(()), - Err(e) => Err(e.into()), - } -} - -async fn store_result(state: &AppState, result: &StoredResult) -> Result<(), ServiceError> { - let json = serde_json::to_string(result).map_err(|e| ServiceError::internal(format!("serialize result: {e}")))?; - let key = result_key(state, &result.job_id); - let channel = result_channel(state, &result.job_id); - let ttl = state.cfg.result_ttl_secs.min(i64::MAX as u64) as i64; - let _: () = state.redis.set(key, json, Some(Expiration::EX(ttl)), None::, false).await?; - let _: i64 = state.redis.publish(channel, "done").await?; - Ok(()) -} - -async fn load_result(state: &AppState, job_id: &str) -> Result, ServiceError> { - let raw: Option = state.redis.get(result_key(state, job_id)).await?; - raw.map(|s| serde_json::from_str(&s).map_err(|e| ServiceError::internal(format!("parse result: {e}")))).transpose() -} - -fn result_to_response(result: StoredResult, created_at_ms: u64) -> Result { - let status = StatusCode::from_u16(result.http_status).unwrap_or(StatusCode::OK); - let body = base64::engine::general_purpose::STANDARD.decode(result.body_base64).map_err(|e| ServiceError::internal(format!("decode result body: {e}")))?; - let mut resp = (status, body).into_response(); - for (name, value) in result.headers { - if let (Ok(name), Ok(value)) = (HeaderName::try_from(name.as_str()), HeaderValue::from_str(&value)) { - resp.headers_mut().insert(name, value); - } - } - resp.headers_mut().insert("x-job-id", header_value(&result.job_id)?); - resp.headers_mut().insert("x-queue-wait-ms", header_value(&now_ms().saturating_sub(created_at_ms).to_string())?); - Ok(resp) -} - -async fn store_body(state: &AppState, job_id: &str, body: Body) -> Result { - let object_ref = format!("{}/{}/body.bin", state.cfg.object_store_prefix.trim_matches('/'), sanitize_key(job_id)); - let mut stream = body.into_data_stream(); - let mut pending = Vec::new(); - let mut total_size = 0usize; - let mut upload_id = None; - let mut parts = Vec::new(); - let part_size = state.cfg.object_multipart_part_size.max(5 * 1024 * 1024); - - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|e| ServiceError::bad_request(format!("read request body: {e}")))?; - total_size = total_size.checked_add(chunk.len()).ok_or_else(|| ServiceError::payload_too_large("request body is too large"))?; - if total_size > state.cfg.max_body_bytes { - abort_upload_if_needed(state, &object_ref, upload_id.as_deref()).await; - return Err(ServiceError::payload_too_large(format!("request body exceeds max size {}", state.cfg.max_body_bytes))); - } - - if upload_id.is_none() { - if state.cfg.object_store_endpoint.is_some() && pending.len() + chunk.len() > state.cfg.inline_threshold { - pending.extend_from_slice(&chunk); - match initiate_multipart_upload(state, &object_ref).await { - Ok(id) => upload_id = Some(id), - Err(e) => return Err(e), - } - } else { - pending.extend_from_slice(&chunk); - continue; - } - } else { - pending.extend_from_slice(&chunk); - } - - if let Some(upload_id) = upload_id.as_deref() { - while pending.len() >= part_size { - let part_body = pending.drain(..part_size).collect::>(); - match upload_multipart_part(state, &object_ref, upload_id, parts.len() + 1, part_body).await { - Ok(part) => parts.push(part), - Err(e) => { - abort_upload_if_needed(state, &object_ref, Some(upload_id)).await; - return Err(e); - } - } - } - } - } - - if let Some(upload_id) = upload_id.as_deref() { - if !pending.is_empty() || parts.is_empty() { - match upload_multipart_part(state, &object_ref, upload_id, parts.len() + 1, pending).await { - Ok(part) => parts.push(part), - Err(e) => { - abort_upload_if_needed(state, &object_ref, Some(upload_id)).await; - return Err(e); - } - } - } - if let Err(e) = complete_multipart_upload(state, &object_ref, upload_id, &parts).await { - abort_upload_if_needed(state, &object_ref, Some(upload_id)).await; - return Err(e); - } - state.metrics.object_offload_total.fetch_add(1, Ordering::Relaxed); - return Ok(BodyLocation { - body_base64: String::new(), - object_ref, - size: total_size, - storage: "object", - }); - } - - Ok(BodyLocation { - body_base64: base64::engine::general_purpose::STANDARD.encode(&pending), - object_ref: String::new(), - size: total_size, - storage: "inline", - }) -} - -async fn load_body(state: &AppState, fields: &HashMap) -> Result, ServiceError> { - let storage = field_string(fields, "storage").unwrap_or_else(|| "inline".to_string()); - if storage == "object" { - let object_ref = field_string(fields, "ref").ok_or_else(|| ServiceError::bad_request("job body is missing object ref"))?; - let url = object_url(state, &object_ref); - let mut req = state.http.get(url); - if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { - req = req.header(name, value); - } - return Ok(req.send().await?.error_for_status()?.bytes().await?.to_vec()); - } - - if let Some(body_base64) = field_string(fields, "body") { - return base64::engine::general_purpose::STANDARD.decode(body_base64).map_err(|e| ServiceError::bad_request(format!("decode job body: {e}"))); - } - Ok(field_bytes(fields, "body").unwrap_or_default()) -} - -async fn initiate_multipart_upload(state: &AppState, object_ref: &str) -> Result { - let url = object_url_with_query(state, object_ref, "uploads"); - let mut req = state.http.post(url); - if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { - req = req.header(name, value); - } - let body = req.send().await?.error_for_status()?.text().await?; - extract_xml_tag(&body, "UploadId").ok_or_else(|| ServiceError::internal("multipart initiate response missing UploadId")) -} - -async fn upload_multipart_part(state: &AppState, object_ref: &str, upload_id: &str, part_number: usize, body: Vec) -> Result { - let query = format!("partNumber={part_number}&uploadId={}", encode_query_component(upload_id)); - let url = object_url_with_query(state, object_ref, &query); - let mut req = state.http.put(url).body(body); - if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { - req = req.header(name, value); - } - let resp = req.send().await?.error_for_status()?; - let etag = resp - .headers() - .get("etag") - .and_then(|value| value.to_str().ok()) - .map(ToOwned::to_owned) - .ok_or_else(|| ServiceError::internal("multipart upload part response missing ETag"))?; - Ok(CompletedPart { part_number, etag }) -} - -async fn complete_multipart_upload(state: &AppState, object_ref: &str, upload_id: &str, parts: &[CompletedPart]) -> Result<(), ServiceError> { - let query = format!("uploadId={}", encode_query_component(upload_id)); - let url = object_url_with_query(state, object_ref, &query); - let body = complete_multipart_xml(parts); - let mut req = state.http.post(url).header("content-type", "application/xml").body(body); - if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { - req = req.header(name, value); - } - req.send().await?.error_for_status()?; - Ok(()) -} - -async fn abort_multipart_upload(state: &AppState, object_ref: &str, upload_id: &str) -> Result<(), ServiceError> { - let query = format!("uploadId={}", encode_query_component(upload_id)); - let url = object_url_with_query(state, object_ref, &query); - let mut req = state.http.delete(url); - if let Some((name, value)) = object_auth_header(&state.cfg.object_store_auth_header)? { - req = req.header(name, value); - } - req.send().await?.error_for_status()?; - Ok(()) -} - -async fn abort_upload_if_needed(state: &AppState, object_ref: &str, upload_id: Option<&str>) { - let Some(upload_id) = upload_id else { - return; - }; - state.metrics.object_multipart_abort_total.fetch_add(1, Ordering::Relaxed); - if let Err(abort_err) = abort_multipart_upload(state, object_ref, upload_id).await { - tracing::warn!(object_ref = %object_ref, upload_id = %upload_id, error = %abort_err.message, "multipart upload abort failed"); - } -} - -fn complete_multipart_xml(parts: &[CompletedPart]) -> String { - let mut out = String::from(""); - for part in parts { - out.push_str(""); - out.push_str(""); - out.push_str(&part.part_number.to_string()); - out.push_str(""); - out.push_str(""); - out.push_str(&xml_escape(&part.etag)); - out.push_str(""); - out.push_str(""); - } - out.push_str(""); - out -} - -fn object_url(state: &AppState, object_ref: &str) -> String { - format!( - "{}/{}/{}", - state.cfg.object_store_endpoint.as_deref().unwrap_or_default().trim_end_matches('/'), - state.cfg.object_store_bucket.trim_matches('/'), - object_ref.trim_start_matches('/') - ) -} - -fn object_url_with_query(state: &AppState, object_ref: &str, query: &str) -> String { - format!("{}?{}", object_url(state, object_ref), query) -} - -fn object_auth_header(raw: &Option) -> Result, ServiceError> { - let Some(raw) = raw.as_deref() else { - return Ok(None); - }; - let Some((name, value)) = raw.split_once(':') else { - return Err(ServiceError::bad_request("AI_OBJECT_STORE_AUTH_HEADER must be `Header-Name: value`")); - }; - if HeaderName::try_from(name.trim()).is_err() || HeaderValue::from_str(value.trim()).is_err() { - return Err(ServiceError::bad_request("invalid object auth header")); - } - Ok(Some((name.trim().to_string(), value.trim().to_string()))) -} - -fn extract_xml_tag(xml: &str, tag: &str) -> Option { - let start_tag = format!("<{tag}>"); - let end_tag = format!(""); - let start = xml.find(&start_tag)? + start_tag.len(); - let end = xml[start..].find(&end_tag)? + start; - Some(xml[start..end].trim().to_string()) -} - -fn encode_query_component(input: &str) -> String { - let mut out = String::with_capacity(input.len()); - for byte in input.bytes() { - if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { - out.push(byte as char); - } else { - out.push_str(&format!("%{byte:02X}")); - } - } - out -} - -fn xml_escape(input: &str) -> String { - input.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """).replace('\'', "'") -} - -async fn trim_stream(state: &AppState, stream: &str) -> Result<(), ServiceError> { - if state.cfg.stream_max_len > 0 { - let _: i64 = state.redis.xtrim(stream, ("MAXLEN", "~", state.cfg.stream_max_len as i64)).await?; - } - Ok(()) -} - -async fn pending_size(state: &AppState, stream: &str) -> i64 { - pending_size_for_group(state, stream, state.cfg.consumer_group.as_str()).await -} - -async fn pending_size_for_group(state: &AppState, stream: &str, group: &str) -> i64 { - let raw: FredResult = state.redis.xpending(stream, group, ()).await; - match raw { - Ok(value) => pending_count_from_value(&value), - Err(e) => { - tracing::debug!(stream = %stream, group = %group, error = %e, "read stream pending size failed"); - 0 - } - } -} - -fn pending_count_from_value(value: &Value) -> i64 { - match value { - Value::Integer(value) => (*value).max(0), - Value::String(value) => value.parse::().unwrap_or(0).max(0), - Value::Bytes(value) => std::str::from_utf8(value).ok().and_then(|value| value.parse::().ok()).unwrap_or(0).max(0), - Value::Array(values) => values.first().map(pending_count_from_value).unwrap_or(0), - Value::Map(values) => values - .iter() - .find_map(|(key, value)| { - let key = key.as_str()?; - if key.eq_ignore_ascii_case("pending") || key.eq_ignore_ascii_case("count") { - Some(pending_count_from_value(value)) - } else { - None - } - }) - .unwrap_or(0), - _ => 0, - } -} - -fn observe_enqueue_latency(metrics: &Metrics, elapsed_ms: u64) { - metrics.enqueue_latency_count.fetch_add(1, Ordering::Relaxed); - metrics.enqueue_latency_sum_ms.fetch_add(elapsed_ms, Ordering::Relaxed); - if elapsed_ms <= 100 { - metrics.enqueue_latency_le_100_ms.fetch_add(1, Ordering::Relaxed); - } else if elapsed_ms <= 500 { - metrics.enqueue_latency_le_500_ms.fetch_add(1, Ordering::Relaxed); - } else if elapsed_ms <= 1000 { - metrics.enqueue_latency_le_1000_ms.fetch_add(1, Ordering::Relaxed); - } else { - metrics.enqueue_latency_gt_1000_ms.fetch_add(1, Ordering::Relaxed); - } -} - -fn observe_body_size(metrics: &Metrics, size: usize) { - metrics.body_size_count.fetch_add(1, Ordering::Relaxed); - metrics.body_size_sum_bytes.fetch_add(size as u64, Ordering::Relaxed); - if size <= 10 * 1024 { - metrics.body_size_le_10kb.fetch_add(1, Ordering::Relaxed); - } else if size <= 128 * 1024 { - metrics.body_size_le_128kb.fetch_add(1, Ordering::Relaxed); - } else if size <= 5 * 1024 * 1024 { - metrics.body_size_le_5mb.fetch_add(1, Ordering::Relaxed); - } else { - metrics.body_size_gt_5mb.fetch_add(1, Ordering::Relaxed); - } -} - -fn observe_worker_processing(metrics: &Metrics, elapsed_ms: u64) { - metrics.worker_processing_count.fetch_add(1, Ordering::Relaxed); - metrics.worker_processing_sum_ms.fetch_add(elapsed_ms, Ordering::Relaxed); - if elapsed_ms <= 1000 { - metrics.worker_processing_le_1000_ms.fetch_add(1, Ordering::Relaxed); - } else if elapsed_ms <= 5000 { - metrics.worker_processing_le_5000_ms.fetch_add(1, Ordering::Relaxed); - } else if elapsed_ms <= 30_000 { - metrics.worker_processing_le_30000_ms.fetch_add(1, Ordering::Relaxed); - } else { - metrics.worker_processing_gt_30000_ms.fetch_add(1, Ordering::Relaxed); - } -} - -fn stream_for_request(state: &AppState, headers: &HeaderMap) -> String { - if !state.cfg.enable_priority_streams { - return state.cfg.stream_key.clone(); - } - match optional_header(headers, "x-queue-priority").as_deref() { - Some("high") => state.cfg.high_priority_stream_key.clone(), - Some("low") => state.cfg.low_priority_stream_key.clone(), - _ => state.cfg.stream_key.clone(), - } -} - -fn worker_streams(state: &AppState) -> Vec { - if state.cfg.enable_priority_streams { - vec![ - state.cfg.high_priority_stream_key.clone(), - state.cfg.stream_key.clone(), - state.cfg.low_priority_stream_key.clone(), - ] - } else { - vec![state.cfg.stream_key.clone()] - } -} - -fn validate_callback_url(state: &AppState, policy: QueuePolicy, callback_url: &str) -> Result<(), ServiceError> { - if policy == QueuePolicy::Queue && callback_url.is_empty() { - return Err(ServiceError::bad_request("missing required header `x-callback-url` for queue policy")); - } - if !callback_url.is_empty() && state.cfg.require_https_callback && !callback_url.starts_with("https://") { - return Err(ServiceError::bad_request("x-callback-url must use https")); - } - Ok(()) -} - -async fn tenant_rate_limit(state: &AppState, tenant: &str) -> Result<(u64, u64), ServiceError> { - let key = format!("{}{}", state.cfg.tenant_rate_limit_prefix, sanitize_key(tenant)); - let rps: Option = state.redis.get(format!("{key}:rps")).await.unwrap_or(None); - let burst: Option = state.redis.get(format!("{key}:burst")).await.unwrap_or(None); - Ok(( - rps.and_then(|v| v.parse().ok()).unwrap_or(state.cfg.rate_limit_rps), - burst.and_then(|v| v.parse().ok()).unwrap_or(state.cfg.rate_limit_burst), - )) -} - -fn required_header(headers: &HeaderMap, name: &str) -> Result { - optional_header(headers, name).ok_or_else(|| ServiceError::bad_request(format!("missing required header `{name}`"))) -} - -fn optional_header(headers: &HeaderMap, name: &str) -> Option { - headers.get(name).and_then(|value| value.to_str().ok()).map(str::trim).filter(|value| !value.is_empty()).map(ToOwned::to_owned) -} - -fn headers_to_json(headers: &HeaderMap) -> Result { - let mut out = HashMap::new(); - for (name, value) in headers { - if let Ok(value) = value.to_str() { - out.insert(name.as_str().to_string(), value.to_string()); - } - } - serde_json::to_string(&out).map_err(|e| ServiceError::internal(format!("serialize headers: {e}"))) -} - -fn should_forward_header(name: &str) -> bool { - let name = name.to_ascii_lowercase(); - !matches!( - name.as_str(), - "host" | "connection" | "content-length" | "transfer-encoding" | "x-original-method" | "x-original-path" | "x-ratelimit-policy" | "x-callback-url" | "x-request-timeout" - ) -} - -fn header_value(value: &str) -> Result { - HeaderValue::from_str(value).map_err(|e| ServiceError::internal(format!("invalid response header value: {e}"))) -} - -fn result_key(state: &AppState, job_id: &str) -> String { - format!("{}{}", state.cfg.result_key_prefix, job_id) -} - -fn result_channel(state: &AppState, job_id: &str) -> String { - format!("{}{}", state.cfg.result_channel_prefix, job_id) -} - -fn new_job_id() -> String { - let now = now_ms(); - let seq = JOB_COUNTER.fetch_add(1, Ordering::Relaxed); - format!("{now:x}{seq:x}") -} - -fn now_ms() -> u64 { - SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64 -} - -fn sanitize_key(input: &str) -> String { - input.chars().map(|ch| if ch.is_ascii_alphanumeric() || matches!(ch, ':' | '_' | '-' | '.') { ch } else { '_' }).collect() -} - -fn build_redis_client(url: &str) -> Result { - let config = Config::from_url(url)?; - Builder::from_config(config).build() -} - -fn build_subscriber_client(url: &str) -> Result { - let config = Config::from_url(url)?; - Builder::from_config(config).build_subscriber_client() -} - -fn field_string(fields: &HashMap, key: &str) -> Option { - fields.get(key).and_then(|value| match value { - Value::String(value) => Some(value.to_string()), - Value::Bytes(value) => String::from_utf8(value.to_vec()).ok(), - Value::Integer(value) => Some(value.to_string()), - _ => None, - }) -} - -fn field_bytes(fields: &HashMap, key: &str) -> Option> { - fields.get(key).and_then(|value| match value { - Value::Bytes(value) => Some(value.to_vec()), - Value::String(value) => Some(value.as_bytes().to_vec()), - _ => None, - }) -} - -fn field_u64(fields: &HashMap, key: &str) -> Option { - fields.get(key).and_then(|value| match value { - Value::Integer(value) => (*value).try_into().ok(), - Value::String(value) => value.parse().ok(), - Value::Bytes(value) => std::str::from_utf8(value).ok().and_then(|value| value.parse().ok()), - _ => None, - }) -} - -fn field_u32(fields: &HashMap, key: &str) -> Option { - field_u64(fields, key).and_then(|value| value.try_into().ok()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn extracts_upload_id_from_multipart_xml() { - let xml = "a+b/c="; - assert_eq!(extract_xml_tag(xml, "UploadId").as_deref(), Some("a+b/c=")); - } - - #[test] - fn encodes_upload_id_for_query_string() { - assert_eq!(encode_query_component("a+b/c="), "a%2Bb%2Fc%3D"); - } - - #[test] - fn builds_complete_multipart_xml_with_escaped_etags() { - let parts = vec![ - CompletedPart { - part_number: 1, - etag: "\"abc&1\"".to_string(), - }, - CompletedPart { - part_number: 2, - etag: "\"def\"".to_string(), - }, - ]; - let xml = complete_multipart_xml(&parts); - assert!(xml.contains("1"abc&1"")); - assert!(xml.contains("2"def"")); - } - - #[test] - fn callback_retry_delay_uses_exponential_backoff_with_cap() { - assert_eq!(callback_retry_delay_ms(1000, 60_000, 1), 1000); - assert_eq!(callback_retry_delay_ms(1000, 60_000, 3), 4000); - assert_eq!(callback_retry_delay_ms(1000, 5000, 8), 5000); - } - - #[test] - fn parses_xpending_summary_count() { - let value = Value::Array(vec![Value::Integer(7), Value::String("0-1".into()), Value::String("0-2".into())]); - assert_eq!(pending_count_from_value(&value), 7); - } - - #[test] - fn observes_histogram_buckets_as_non_overlapping_counts() { - let metrics = Metrics::default(); - observe_enqueue_latency(&metrics, 80); - observe_enqueue_latency(&metrics, 800); - observe_body_size(&metrics, 8 * 1024); - observe_body_size(&metrics, 256 * 1024); - observe_worker_processing(&metrics, 2000); - - assert_eq!(metrics.enqueue_latency_count.load(Ordering::Relaxed), 2); - assert_eq!(metrics.enqueue_latency_le_100_ms.load(Ordering::Relaxed), 1); - assert_eq!(metrics.enqueue_latency_le_1000_ms.load(Ordering::Relaxed), 1); - assert_eq!(metrics.body_size_count.load(Ordering::Relaxed), 2); - assert_eq!(metrics.body_size_le_10kb.load(Ordering::Relaxed), 1); - assert_eq!(metrics.body_size_le_5mb.load(Ordering::Relaxed), 1); - assert_eq!(metrics.worker_processing_count.load(Ordering::Relaxed), 1); - assert_eq!(metrics.worker_processing_le_5000_ms.load(Ordering::Relaxed), 1); - } + app::run().await } diff --git a/plugins/wasm/Cargo.toml b/plugins/wasm/Cargo.toml index 81063f30..c7aa5bec 100644 --- a/plugins/wasm/Cargo.toml +++ b/plugins/wasm/Cargo.toml @@ -12,6 +12,7 @@ publish = false [workspace.dependencies] proxy-wasm = "0.2" +serde_json = "1" [profile.release] codegen-units = 1 diff --git a/plugins/wasm/ai-gateway-queue/Cargo.toml b/plugins/wasm/ai-gateway-queue/Cargo.toml index 1c515529..b470a411 100644 --- a/plugins/wasm/ai-gateway-queue/Cargo.toml +++ b/plugins/wasm/ai-gateway-queue/Cargo.toml @@ -10,3 +10,4 @@ crate-type = ["cdylib"] [dependencies] proxy-wasm.workspace = true +serde_json.workspace = true diff --git a/plugins/wasm/ai-gateway-queue/README.md b/plugins/wasm/ai-gateway-queue/README.md index d325e351..50bdd208 100644 --- a/plugins/wasm/ai-gateway-queue/README.md +++ b/plugins/wasm/ai-gateway-queue/README.md @@ -102,13 +102,32 @@ AI_REQUIRE_HTTPS_CALLBACK=false "max_pending_calls": 1 }, "plugin_config": { - "service_cluster": "ai-gateway-service", - "service_authority": "ai-gateway-service", - "rate_limit_path": "/v1/ratelimit/check", - "enqueue_path": "/v1/queue/enqueue", - "wait_path": "/v1/queue/enqueue-and-wait", - "service_timeout_ms": 65000, - "require_policy": true + "service": { + "cluster": "ai-gateway-service", + "authority": "ai-gateway-service", + "timeout_ms": 65000 + }, + "paths": { + "rate_limit": "/v1/ratelimit/check", + "enqueue": "/v1/queue/enqueue", + "wait": "/v1/queue/enqueue-and-wait" + }, + "headers": { + "policy": "x-ratelimit-policy", + "tenant": "x-tenant-id", + "model": "x-model", + "priority": "x-queue-priority" + }, + "policies": { + "require": true, + "default": null + }, + "priority": { + "enabled": true, + "default": "normal", + "high_models": ["gpt-4o"], + "low_tenants": ["free"] + } }, "clusters": { "ai-gateway-service": "http://127.0.0.1:18080" @@ -125,6 +144,11 @@ AI_REQUIRE_HTTPS_CALLBACK=false - `wait_path`：入队并等待接口 - `service_timeout_ms`：调用外部服务超时 - `require_policy`：是否强制要求请求头携带策略 +- `headers.*`：自定义客户端侧策略、租户、模型、优先级 header；插件会转成外部服务统一使用的 `x-ratelimit-policy`、`x-tenant-id`、`x-model`、`x-queue-priority` +- `policies.default`：未携带策略 header 时使用的默认策略；为空且 `require=true` 时会返回 `400` +- `priority.*`：插件侧优先级推导规则，支持按模型或租户自动设置 `high` / `low` + +插件配置优先支持上面的结构化 JSON；旧的扁平字段仍兼容，例如 `service_cluster`、`rate_limit_path`、`tenant_header`、`default_policy`、`high_priority_models`。 ## 请求头 @@ -196,6 +220,8 @@ curl -i http://localhost:9080/your/api \ ## 生产化能力 - Redis Stream 支持 `MAXLEN ~` 裁剪，通过 `AI_QUEUE_MAX_LEN` 控制 +- 租户限流支持按租户、模型、路由、策略多维覆盖，并支持单请求 cost +- 优先级队列支持 header、模型、租户规则推导，并由 worker 按权重消费高/普通/低优先级 Stream - Worker 崩溃后通过 `XAUTOCLAIM` 重认领 pending job，并通过 Redis 处理租约避免长任务被重复执行 - 回调失败会进入 `AI_CALLBACK_RETRY_STREAM`，按指数退避重试，超过最大次数后进入 `AI_CALLBACK_DLQ_STREAM` - 大 body 可通过 `AI_OBJECT_STORE_ENDPOINT` 走 S3-compatible multipart 卸载，Redis Stream 中只保留 `ref` diff --git a/plugins/wasm/ai-gateway-queue/plugin.yaml b/plugins/wasm/ai-gateway-queue/plugin.yaml index 6404f60e..49a04291 100644 --- a/plugins/wasm/ai-gateway-queue/plugin.yaml +++ b/plugins/wasm/ai-gateway-queue/plugin.yaml @@ -16,3 +16,10 @@ spec: enqueue_path: /v1/queue/enqueue wait_path: /v1/queue/enqueue-and-wait service_timeout_ms: 65000 + require_policy: true + policy_header: x-ratelimit-policy + tenant_header: x-tenant-id + model_header: x-model + priority_header: x-queue-priority + priority_enabled: true + default_priority: normal diff --git a/plugins/wasm/ai-gateway-queue/src/lib.rs b/plugins/wasm/ai-gateway-queue/src/lib.rs index fff8ca6b..84757436 100644 --- a/plugins/wasm/ai-gateway-queue/src/lib.rs +++ b/plugins/wasm/ai-gateway-queue/src/lib.rs @@ -3,6 +3,7 @@ use std::time::Duration; use proxy_wasm::hostcalls; use proxy_wasm::traits::*; use proxy_wasm::types::*; +use serde_json::Value; proxy_wasm::main! {{ proxy_wasm::set_log_level(LogLevel::Info); @@ -18,6 +19,17 @@ struct AiGatewayConfig { wait_path: String, service_timeout_ms: u64, require_policy: bool, + policy_header: String, + tenant_header: String, + model_header: String, + priority_header: String, + default_policy: Option, + priority_enabled: bool, + default_priority: String, + high_priority_models: Vec, + low_priority_models: Vec, + high_priority_tenants: Vec, + low_priority_tenants: Vec, } impl Default for AiGatewayConfig { @@ -30,10 +42,112 @@ impl Default for AiGatewayConfig { wait_path: "/v1/queue/enqueue-and-wait".to_string(), service_timeout_ms: 65_000, require_policy: true, + policy_header: "x-ratelimit-policy".to_string(), + tenant_header: "x-tenant-id".to_string(), + model_header: "x-model".to_string(), + priority_header: "x-queue-priority".to_string(), + default_policy: None, + priority_enabled: true, + default_priority: "normal".to_string(), + high_priority_models: Vec::new(), + low_priority_models: Vec::new(), + high_priority_tenants: Vec::new(), + low_priority_tenants: Vec::new(), } } } +impl AiGatewayConfig { + fn parse(raw: &[u8]) -> Self { + let mut cfg = Self::default(); + if raw.is_empty() { + return cfg; + } + + if let Ok(value) = serde_json::from_slice::(raw) { + cfg.apply_json(&value); + return cfg.normalized(); + } + + let text = String::from_utf8_lossy(raw); + cfg.apply_legacy_lines(&text); + cfg.normalized() + } + + fn apply_json(&mut self, value: &Value) { + set_string(value, &["service_cluster"], &mut self.service_cluster); + set_string(value, &["service", "cluster"], &mut self.service_cluster); + set_string(value, &["service_authority"], &mut self.service_authority); + set_string(value, &["service", "authority"], &mut self.service_authority); + set_string(value, &["rate_limit_path"], &mut self.rate_limit_path); + set_string(value, &["paths", "rate_limit"], &mut self.rate_limit_path); + set_string(value, &["enqueue_path"], &mut self.enqueue_path); + set_string(value, &["paths", "enqueue"], &mut self.enqueue_path); + set_string(value, &["wait_path"], &mut self.wait_path); + set_string(value, &["paths", "wait"], &mut self.wait_path); + set_u64(value, &["service_timeout_ms"], &mut self.service_timeout_ms); + set_u64(value, &["service", "timeout_ms"], &mut self.service_timeout_ms); + set_bool(value, &["require_policy"], &mut self.require_policy); + set_bool(value, &["policies", "require"], &mut self.require_policy); + self.default_policy = string_at(value, &["default_policy"]).or_else(|| string_at(value, &["policies", "default"])); + set_string(value, &["policy_header"], &mut self.policy_header); + set_string(value, &["headers", "policy"], &mut self.policy_header); + set_string(value, &["tenant_header"], &mut self.tenant_header); + set_string(value, &["headers", "tenant"], &mut self.tenant_header); + set_string(value, &["model_header"], &mut self.model_header); + set_string(value, &["headers", "model"], &mut self.model_header); + set_string(value, &["priority_header"], &mut self.priority_header); + set_string(value, &["headers", "priority"], &mut self.priority_header); + set_bool(value, &["priority_enabled"], &mut self.priority_enabled); + set_bool(value, &["priority", "enabled"], &mut self.priority_enabled); + set_string(value, &["default_priority"], &mut self.default_priority); + set_string(value, &["priority", "default"], &mut self.default_priority); + self.high_priority_models = string_vec_at(value, &["priority", "high_models"]).or_else(|| string_vec_at(value, &["high_priority_models"])).unwrap_or_default(); + self.low_priority_models = string_vec_at(value, &["priority", "low_models"]).or_else(|| string_vec_at(value, &["low_priority_models"])).unwrap_or_default(); + self.high_priority_tenants = string_vec_at(value, &["priority", "high_tenants"]).or_else(|| string_vec_at(value, &["high_priority_tenants"])).unwrap_or_default(); + self.low_priority_tenants = string_vec_at(value, &["priority", "low_tenants"]).or_else(|| string_vec_at(value, &["low_priority_tenants"])).unwrap_or_default(); + } + + fn apply_legacy_lines(&mut self, text: &str) { + for line in text.lines() { + let Some((key, value)) = line.split_once(':') else { continue }; + let key = key.trim().trim_matches(['"', '\'', '{', ',', ' '].as_ref()); + let value = value.trim().trim_matches(['"', '\'', ',', ' '].as_ref()); + match key { + "service_cluster" => self.service_cluster = value.to_string(), + "service_authority" => self.service_authority = value.to_string(), + "rate_limit_path" => self.rate_limit_path = value.to_string(), + "enqueue_path" => self.enqueue_path = value.to_string(), + "wait_path" => self.wait_path = value.to_string(), + "service_timeout_ms" => self.service_timeout_ms = value.parse().unwrap_or(self.service_timeout_ms), + "require_policy" => self.require_policy = value.parse().unwrap_or(self.require_policy), + "policy_header" => self.policy_header = value.to_string(), + "tenant_header" => self.tenant_header = value.to_string(), + "model_header" => self.model_header = value.to_string(), + "priority_header" => self.priority_header = value.to_string(), + "default_policy" => self.default_policy = Some(value.to_string()), + "priority_enabled" => self.priority_enabled = value.parse().unwrap_or(self.priority_enabled), + "default_priority" => self.default_priority = value.to_string(), + "high_priority_models" => self.high_priority_models = parse_csv(value), + "low_priority_models" => self.low_priority_models = parse_csv(value), + "high_priority_tenants" => self.high_priority_tenants = parse_csv(value), + "low_priority_tenants" => self.low_priority_tenants = parse_csv(value), + _ => {} + } + } + } + + fn normalized(mut self) -> Self { + self.policy_header = normalize_header_name(&self.policy_header, "x-ratelimit-policy"); + self.tenant_header = normalize_header_name(&self.tenant_header, "x-tenant-id"); + self.model_header = normalize_header_name(&self.model_header, "x-model"); + self.priority_header = normalize_header_name(&self.priority_header, "x-queue-priority"); + self.default_priority = normalize_priority(&self.default_priority).unwrap_or_else(|| "normal".to_string()); + self.default_policy = self.default_policy.and_then(|value| normalize_policy(&value)); + self + } +} + #[derive(Default)] struct AiGatewayRoot { cfg: AiGatewayConfig, @@ -49,24 +163,7 @@ impl RootContext for AiGatewayRoot { fn on_configure(&mut self, _: usize) -> bool { let raw = self.get_plugin_configuration().unwrap_or_default(); - let text = String::from_utf8_lossy(&raw); - let mut cfg = AiGatewayConfig::default(); - for line in text.lines() { - let Some((key, value)) = line.split_once(':') else { continue }; - let key = key.trim().trim_matches(['"', '\'', '{', ',', ' '].as_ref()); - let value = value.trim().trim_matches(['"', '\'', ',', ' '].as_ref()); - match key { - "service_cluster" => cfg.service_cluster = value.to_string(), - "service_authority" => cfg.service_authority = value.to_string(), - "rate_limit_path" => cfg.rate_limit_path = value.to_string(), - "enqueue_path" => cfg.enqueue_path = value.to_string(), - "wait_path" => cfg.wait_path = value.to_string(), - "service_timeout_ms" => cfg.service_timeout_ms = value.parse().unwrap_or(cfg.service_timeout_ms), - "require_policy" => cfg.require_policy = value.parse().unwrap_or(true), - _ => {} - } - } - self.cfg = cfg; + self.cfg = AiGatewayConfig::parse(&raw); true } @@ -130,7 +227,7 @@ impl HttpContext for AiGatewayHttp { return Action::Continue; }; - if self.get_http_request_header("x-tenant-id").is_none() { + if self.tenant_id().is_none() { self.send_json(400, r#"{"error":"missing_x_tenant_id"}"#); return Action::Pause; } @@ -187,7 +284,8 @@ impl HttpContext for AiGatewayHttp { impl AiGatewayHttp { fn request_policy(&self) -> Option { - match self.get_http_request_header("x-ratelimit-policy").as_deref().map(str::trim) { + let value = self.get_http_request_header(&self.cfg.policy_header).or_else(|| self.cfg.default_policy.clone())?; + match normalize_policy(&value).as_deref() { Some("abandon") => Some(Policy::Abandon), Some("queue") => Some(Policy::Queue), Some("wait") => Some(Policy::Wait), @@ -211,6 +309,10 @@ impl AiGatewayHttp { } fn service_headers(&self, path: &str) -> Vec<(String, String)> { + let policy = self.request_policy().map(policy_name).unwrap_or("abandon").to_string(); + let tenant_id = self.tenant_id().unwrap_or_default(); + let model = self.model().unwrap_or_else(|| "default".to_string()); + let priority = self.queue_priority(&tenant_id, &model); let mut out = vec![ (":method".to_string(), "POST".to_string()), (":path".to_string(), path.to_string()), @@ -220,7 +322,13 @@ impl AiGatewayHttp { self.get_http_request_header(":method").unwrap_or_else(|| "POST".to_string()), ), ("x-original-path".to_string(), self.get_http_request_header(":path").unwrap_or_else(|| "/".to_string())), + ("x-ratelimit-policy".to_string(), policy), + ("x-tenant-id".to_string(), tenant_id), + ("x-model".to_string(), model), ]; + if let Some(priority) = priority { + out.push(("x-queue-priority".to_string(), priority)); + } for (name, value) in self.get_http_request_headers() { if should_forward_to_service(&name) { @@ -230,6 +338,30 @@ impl AiGatewayHttp { out } + fn tenant_id(&self) -> Option { + self.get_http_request_header(&self.cfg.tenant_header).filter(|value| !value.trim().is_empty()) + } + + fn model(&self) -> Option { + self.get_http_request_header(&self.cfg.model_header).filter(|value| !value.trim().is_empty()) + } + + fn queue_priority(&self, tenant_id: &str, model: &str) -> Option { + if !self.cfg.priority_enabled { + return None; + } + if let Some(priority) = self.get_http_request_header(&self.cfg.priority_header).and_then(|value| normalize_priority(&value)) { + return Some(priority); + } + if contains_value(&self.cfg.high_priority_tenants, tenant_id) || contains_value(&self.cfg.high_priority_models, model) { + return Some("high".to_string()); + } + if contains_value(&self.cfg.low_priority_tenants, tenant_id) || contains_value(&self.cfg.low_priority_models, model) { + return Some("low".to_string()); + } + normalize_priority(&self.cfg.default_priority) + } + fn handle_rate_limit_response(&mut self, body_size: usize) { let status = self.service_status(); let body = self.get_http_call_response_body(0, body_size).unwrap_or_default(); @@ -286,10 +418,27 @@ fn should_forward_to_service(name: &str) -> bool { !lower.starts_with(':') && !matches!( lower.as_str(), - "host" | "connection" | "content-length" | "transfer-encoding" | "x-original-method" | "x-original-path" + "host" + | "connection" + | "content-length" + | "transfer-encoding" + | "x-original-method" + | "x-original-path" + | "x-ratelimit-policy" + | "x-tenant-id" + | "x-model" + | "x-queue-priority" ) } +fn policy_name(policy: Policy) -> &'static str { + match policy { + Policy::Abandon => "abandon", + Policy::Queue => "queue", + Policy::Wait => "wait", + } +} + fn contains_allowed_true(text: &str) -> bool { text.contains(r#""allowed":true"#) || text.contains(r#""allowed": true"#) } @@ -300,3 +449,124 @@ fn extract_json_number(text: &str, key: &str) -> Option { let digits = text[pos + needle.len()..].chars().skip_while(|c| c.is_whitespace()).take_while(|c| c.is_ascii_digit()).collect::(); digits.parse().ok() } + +fn value_at<'a>(value: &'a Value, path: &[&str]) -> Option<&'a Value> { + let mut current = value; + for key in path { + current = current.get(*key)?; + } + Some(current) +} + +fn string_at(value: &Value, path: &[&str]) -> Option { + value_at(value, path).and_then(|value| value.as_str().map(ToOwned::to_owned)) +} + +fn set_string(value: &Value, path: &[&str], target: &mut String) { + if let Some(value) = string_at(value, path).filter(|value| !value.trim().is_empty()) { + *target = value; + } +} + +fn set_u64(value: &Value, path: &[&str], target: &mut u64) { + if let Some(value) = value_at(value, path).and_then(|value| value.as_u64()) { + *target = value; + } +} + +fn set_bool(value: &Value, path: &[&str], target: &mut bool) { + if let Some(value) = value_at(value, path).and_then(|value| value.as_bool()) { + *target = value; + } +} + +fn string_vec_at(value: &Value, path: &[&str]) -> Option> { + let value = value_at(value, path)?; + if let Some(raw) = value.as_str() { + return Some(parse_csv(raw)); + } + let values = value.as_array()?; + Some(values.iter().filter_map(|value| value.as_str().map(ToOwned::to_owned)).collect()) +} + +fn parse_csv(value: &str) -> Vec { + value.split(',').map(str::trim).filter(|value| !value.is_empty()).map(ToOwned::to_owned).collect() +} + +fn normalize_header_name(value: &str, fallback: &str) -> String { + let value = value.trim().to_ascii_lowercase(); + if value.is_empty() || value.starts_with(':') { + fallback.to_string() + } else { + value + } +} + +fn normalize_policy(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "abandon" => Some("abandon".to_string()), + "queue" => Some("queue".to_string()), + "wait" => Some("wait".to_string()), + _ => None, + } +} + +fn normalize_priority(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "high" => Some("high".to_string()), + "normal" | "default" | "medium" => Some("normal".to_string()), + "low" => Some("low".to_string()), + _ => None, + } +} + +fn contains_value(values: &[String], needle: &str) -> bool { + values.iter().any(|value| value.eq_ignore_ascii_case(needle)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_nested_json_config() { + let cfg = AiGatewayConfig::parse( + br#"{ + "service": {"cluster": "svc", "authority": "svc.local", "timeout_ms": 1200}, + "paths": {"rate_limit": "/rl", "enqueue": "/q", "wait": "/w"}, + "headers": {"policy": "X-Policy", "tenant": "X-Org", "model": "X-LLM", "priority": "X-Priority"}, + "policies": {"require": false, "default": "queue"}, + "priority": {"default": "low", "high_models": ["gpt-4"], "low_tenants": "free,basic"} + }"#, + ); + + assert_eq!(cfg.service_cluster, "svc"); + assert_eq!(cfg.service_timeout_ms, 1200); + assert_eq!(cfg.rate_limit_path, "/rl"); + assert_eq!(cfg.policy_header, "x-policy"); + assert!(!cfg.require_policy); + assert_eq!(cfg.default_policy.as_deref(), Some("queue")); + assert_eq!(cfg.default_priority, "low"); + assert_eq!(cfg.high_priority_models, vec!["gpt-4"]); + assert_eq!(cfg.low_priority_tenants, vec!["free", "basic"]); + } + + #[test] + fn parses_legacy_config_lines() { + let cfg = AiGatewayConfig::parse( + br#" +service_cluster: ai-gateway +service_timeout_ms: 3000 +tenant_header: X-Org +default_policy: wait +high_priority_models: qwen-max, deepseek-chat +"#, + ); + + assert_eq!(cfg.service_cluster, "ai-gateway"); + assert_eq!(cfg.service_timeout_ms, 3000); + assert_eq!(cfg.tenant_header, "x-org"); + assert_eq!(cfg.default_policy.as_deref(), Some("wait")); + assert_eq!(cfg.high_priority_models, vec!["qwen-max", "deepseek-chat"]); + } +} From a5092db8c7b908a3eae5d82cc40f1193740abef2 Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Thu, 21 May 2026 17:42:28 +0800 Subject: [PATCH 13/19] feat: add admin UI support for AI queue gateway Expose schema/readme and tenant queue quota admin APIs so the queue gateway can be configured from the frontend while keeping existing runtime behavior compatible. Co-authored-by: Cursor --- binary/ai-gateway-service/Cargo.toml | 3 +- binary/ai-gateway-service/src/app.rs | 5 +- binary/ai-gateway-service/src/app/admin.rs | 278 ++++++++++++++++++ .../ai-gateway-service/src/app/ratelimit.rs | 159 ++++++++++ binary/ai-gateway-service/src/app/runtime.rs | 5 + binary/ai-gateway-service/src/app/types.rs | 214 +++++++++++++- plugins/wasm/ai-gateway-queue/README.md | 12 +- 7 files changed, 668 insertions(+), 8 deletions(-) create mode 100644 binary/ai-gateway-service/src/app/admin.rs diff --git a/binary/ai-gateway-service/Cargo.toml b/binary/ai-gateway-service/Cargo.toml index 00fbae5b..dc316b41 100644 --- a/binary/ai-gateway-service/Cargo.toml +++ b/binary/ai-gateway-service/Cargo.toml @@ -21,9 +21,10 @@ futures-util = { workspace = true } fred = { version = "10.1.0", default-features = false, features = ["enable-rustls", "i-keys", "i-scripts", "i-streams", "subscriber-client", "transactions"] } http = "1" reqwest = { workspace = true, features = ["json"] } +schemars = "0.8" serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } tokio = { workspace = true, features = ["full"] } -tower-http = { version = "0.6", features = ["trace"] } +tower-http = { version = "0.6", features = ["cors", "trace"] } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } diff --git a/binary/ai-gateway-service/src/app.rs b/binary/ai-gateway-service/src/app.rs index 6ca7a07f..6aab3504 100644 --- a/binary/ai-gateway-service/src/app.rs +++ b/binary/ai-gateway-service/src/app.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use axum::body::Body; -use axum::extract::{DefaultBodyLimit, Path, State}; +use axum::extract::{DefaultBodyLimit, Path, Query, State}; use axum::http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode, Uri}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; @@ -17,8 +17,10 @@ use fred::prelude::*; use fred::types::streams::XReadResponse; use fred::types::ExpireOptions; use futures_util::StreamExt; +use schemars::{schema_for, JsonSchema}; use serde::{Deserialize, Serialize}; use tokio::sync::Semaphore; +use tower_http::cors::CorsLayer; use tower_http::trace::TraceLayer; include!("app/types.rs"); @@ -30,5 +32,6 @@ include!("app/result_store.rs"); include!("app/object_store.rs"); include!("app/metrics.rs"); include!("app/ratelimit.rs"); +include!("app/admin.rs"); include!("app/util.rs"); include!("app/tests.rs"); diff --git a/binary/ai-gateway-service/src/app/admin.rs b/binary/ai-gateway-service/src/app/admin.rs new file mode 100644 index 00000000..e60892e4 --- /dev/null +++ b/binary/ai-gateway-service/src/app/admin.rs @@ -0,0 +1,278 @@ +const AI_GATEWAY_QUEUE_PLUGIN: &str = "ai-gateway-queue"; +const AI_GATEWAY_QUEUE_README: &str = include_str!("../../../../plugins/wasm/ai-gateway-queue/README.md"); + +async fn admin_plugin_schema(Path(plugin): Path) -> Result, ServiceError> { + if plugin != AI_GATEWAY_QUEUE_PLUGIN { + return Err(ServiceError::bad_request(format!("unsupported plugin `{plugin}`"))); + } + + let schema = schema_for!(AiGatewayQueuePluginConfig); + let mut value = serde_json::to_value(&schema).map_err(|e| ServiceError::internal(format!("serialize schema: {e}")))?; + add_ai_gateway_queue_schema_extensions(&mut value); + Ok(Json(value)) +} + +async fn admin_plugin_readme(Path(plugin): Path) -> Result { + if plugin != AI_GATEWAY_QUEUE_PLUGIN { + return Err(ServiceError::bad_request(format!("unsupported plugin `{plugin}`"))); + } + Ok((StatusCode::OK, [("content-type", "text/markdown; charset=utf-8")], AI_GATEWAY_QUEUE_README).into_response()) +} + +async fn admin_list_tenant_rate_limits(State(state): State, Query(filters): Query>) -> Result>, ServiceError> { + let rules = list_tenant_rate_limit_rules(&state, &filters).await?; + Ok(Json(rules)) +} + +async fn admin_upsert_tenant_rate_limit(State(state): State, Json(rule): Json) -> Result, ServiceError> { + let rule = upsert_tenant_rate_limit_rule(&state, rule).await?; + Ok(Json(rule)) +} + +async fn admin_delete_tenant_rate_limit(State(state): State, Json(rule): Json) -> Result, ServiceError> { + let removed = delete_tenant_rate_limit_rule(&state, rule).await?; + Ok(Json(serde_json::json!({ "deleted": removed }))) +} + +fn add_ai_gateway_queue_schema_extensions(value: &mut serde_json::Value) { + let example = serde_json::to_string_pretty(&AiGatewayQueuePluginConfig::default()).unwrap_or_default(); + if let Some(object) = value.as_object_mut() { + object.insert("x-example-raw".to_string(), serde_json::Value::String(example)); + object.insert( + "x-title-i18n".to_string(), + serde_json::json!({ + "zh-CN": "AI 请求队列网关", + "en": "AI Request Queue Gateway" + }), + ); + object.insert( + "x-description-i18n".to_string(), + serde_json::json!({ + "zh-CN": "配置 AI 请求队列网关：队列后端接入、入队接口路径、请求头映射、队列模式与优先级路由。", + "en": "Configure the AI request queue gateway: queue backend access, enqueue paths, header mapping, queue mode and priority routing." + }), + ); + } + + let Some(definitions) = value.get_mut("definitions").and_then(|v| v.as_object_mut()) else { return }; + + // 子配置卡片自身的标题/说明：会被 SchemaForm 的 el-card 标题区使用。 + annotate_schema_meta( + definitions.get_mut("AiGatewayServiceConfig"), + "队列后端接入", + "Queue Backend Access", + "Wasm 插件调用外部队列后端时使用的 cluster、authority 和超时设置。", + "Cluster, authority and timeout the wasm plugin uses to call the external queue backend.", + ); + annotate_schema_meta( + definitions.get_mut("AiGatewayPathsConfig"), + "接口路径", + "Paths", + "队列后端暴露的准入判定、入队、入队并等待三类 HTTP 路径。", + "HTTP paths exposed by the queue backend for admission check, enqueue, and enqueue-and-wait.", + ); + annotate_schema_meta( + definitions.get_mut("AiGatewayHeadersConfig"), + "请求头映射", + "Headers", + "客户端实际使用的 Header 名称；插件会把它们统一转成队列后端期望的标准 Header。", + "Header names used by clients; the plugin remaps them to the standard headers the queue backend expects.", + ); + annotate_schema_meta( + definitions.get_mut("AiGatewayPoliciesConfig"), + "队列模式", + "Queue Mode", + "控制 X-RateLimit-Policy 请求头是否必填，以及未携带时使用的默认队列模式（abandon / queue / wait）。", + "Controls whether the X-RateLimit-Policy header is required, and the default queue mode used when it is missing (abandon / queue / wait).", + ); + annotate_schema_meta( + definitions.get_mut("AiGatewayPriorityConfig"), + "优先级路由", + "Priority Routing", + "队列优先级的开关、默认值，以及按模型 / 租户自动选择高 / 普通 / 低优先级队列的规则。", + "Queue priority switch, default value, and per-model / per-tenant rules that route requests into high / normal / low priority streams.", + ); + + set_field_descriptions( + definitions.get_mut("AiGatewayServiceConfig"), + &[ + ( + "cluster", + "队列后端 Cluster", + "Queue Backend Cluster", + "SpaceGate 中指向队列后端的 cluster 名称，对应 SpaceGate 配置里的 clusters 键。", + "Name of the SpaceGate cluster pointing to the queue backend; matches the key under the clusters field.", + ), + ( + "authority", + "队列后端 Authority", + "Queue Backend Authority", + "Wasm 插件 dispatch HTTP call 时使用的 :authority，通常和 cluster 同名。", + "The :authority used by the wasm dispatch_http_call; usually the same as the cluster name.", + ), + ( + "timeout_ms", + "调用超时（毫秒）", + "Timeout (ms)", + "调用队列后端的超时时间；wait 模式需要留足同步等待时间，建议 60000 ms 以上。", + "Timeout for calling the queue backend. Keep it above 60000 ms when wait mode is used.", + ), + ], + ); + + set_field_descriptions( + definitions.get_mut("AiGatewayPathsConfig"), + &[ + ( + "rate_limit", + "准入判定路径", + "Admission Check Path", + "队列后端用于判断请求是否需要入队的准入接口，默认 /v1/ratelimit/check。", + "Backend path that decides whether a request should be enqueued. Default: /v1/ratelimit/check.", + ), + ( + "enqueue", + "入队路径", + "Enqueue Path", + "queue 模式使用的异步入队接口，默认 /v1/queue/enqueue。", + "Endpoint used by the queue (async) mode. Default: /v1/queue/enqueue.", + ), + ( + "wait", + "入队并等待路径", + "Enqueue-and-Wait Path", + "wait 模式使用的入队并同步等待结果接口，默认 /v1/queue/enqueue-and-wait。", + "Endpoint used by the wait (sync) mode. Default: /v1/queue/enqueue-and-wait.", + ), + ], + ); + + set_field_descriptions( + definitions.get_mut("AiGatewayHeadersConfig"), + &[ + ( + "policy", + "队列模式 Header", + "Queue Mode Header", + "客户端用于声明队列模式（abandon / queue / wait）的 Header，插件会转成后端使用的 x-ratelimit-policy。", + "Header the client uses to declare the queue mode (abandon / queue / wait); remapped to x-ratelimit-policy.", + ), + ( + "tenant", + "租户 Header", + "Tenant Header", + "客户端表示租户身份的 Header，插件会转成队列后端使用的 x-tenant-id。", + "Header carrying tenant identity; remapped to x-tenant-id for the queue backend.", + ), + ( + "model", + "模型 Header", + "Model Header", + "客户端声明目标模型的 Header，会被透传为队列后端使用的 x-model。", + "Header that names the target model; remapped to x-model for the queue backend.", + ), + ( + "priority", + "优先级 Header", + "Priority Header", + "客户端可选的队列优先级 Header，启用优先级时会被转为 x-queue-priority（取值 high/normal/low）。", + "Optional header for queue priority, remapped to x-queue-priority (values high/normal/low).", + ), + ], + ); + + set_field_descriptions( + definitions.get_mut("AiGatewayPoliciesConfig"), + &[ + ( + "require", + "强制要求队列模式 Header", + "Require Queue Mode Header", + "为 true 时，请求未携带队列模式 Header 会直接返回 400；关闭后会回退到默认队列模式。", + "When true, requests without the queue-mode header are rejected with 400; otherwise falls back to the default mode.", + ), + ( + "default", + "默认队列模式", + "Default Queue Mode", + "未携带队列模式 Header 且 require 为 false 时使用的默认模式，可选 abandon / queue / wait。", + "Default queue mode when require is false and the request omits the header. One of abandon / queue / wait.", + ), + ], + ); + + set_field_descriptions( + definitions.get_mut("AiGatewayPriorityConfig"), + &[ + ( + "enabled", + "启用优先级路由", + "Enable Priority Routing", + "总开关：关闭后所有请求都进入 normal 优先级队列，不再读取模型/租户规则。", + "Master switch; when disabled, all requests go to the normal-priority queue and per-model / per-tenant rules are ignored.", + ), + ( + "default", + "默认队列优先级", + "Default Queue Priority", + "命中不到任何规则时使用的默认队列优先级，可选 high / normal / low。", + "Default queue priority used when no rule matches. One of high / normal / low.", + ), + ( + "high_models", + "高优队列模型列表", + "High Priority Models", + "命中后自动路由到高优队列的模型名列表（精确匹配，区分大小写）。", + "Models that are routed to the high-priority queue (exact, case-sensitive match).", + ), + ( + "low_models", + "低优队列模型列表", + "Low Priority Models", + "命中后自动路由到低优队列的模型名列表。", + "Models that are routed to the low-priority queue.", + ), + ( + "high_tenants", + "高优队列租户列表", + "High Priority Tenants", + "命中后自动路由到高优队列的租户 ID 列表。", + "Tenant IDs that are routed to the high-priority queue.", + ), + ( + "low_tenants", + "低优队列租户列表", + "Low Priority Tenants", + "命中后自动路由到低优队列的租户 ID 列表，常用于免费 / 试用租户。", + "Tenant IDs that are routed to the low-priority queue, typically used for free or trial tenants.", + ), + ], + ); +} + +fn annotate_schema_meta(schema: Option<&mut serde_json::Value>, zh_title: &str, en_title: &str, zh_desc: &str, en_desc: &str) { + let Some(object) = schema.and_then(|v| v.as_object_mut()) else { return }; + object.insert( + "x-title-i18n".to_string(), + serde_json::json!({ "zh-CN": zh_title, "en": en_title }), + ); + object.insert( + "x-description-i18n".to_string(), + serde_json::json!({ "zh-CN": zh_desc, "en": en_desc }), + ); +} + +fn set_field_descriptions(schema: Option<&mut serde_json::Value>, items: &[(&str, &str, &str, &str, &str)]) { + let Some(properties) = schema.and_then(|v| v.get_mut("properties")).and_then(|v| v.as_object_mut()) else { return }; + for (key, zh_title, en_title, zh_desc, en_desc) in items { + let Some(field) = properties.get_mut(*key).and_then(|v| v.as_object_mut()) else { continue }; + field.insert( + "x-title-i18n".to_string(), + serde_json::json!({ "zh-CN": zh_title, "en": en_title }), + ); + field.insert( + "x-description-i18n".to_string(), + serde_json::json!({ "zh-CN": zh_desc, "en": en_desc }), + ); + } +} diff --git a/binary/ai-gateway-service/src/app/ratelimit.rs b/binary/ai-gateway-service/src/app/ratelimit.rs index 48ed9535..6da10542 100644 --- a/binary/ai-gateway-service/src/app/ratelimit.rs +++ b/binary/ai-gateway-service/src/app/ratelimit.rs @@ -50,3 +50,162 @@ fn parse_tenant_rate_limit(raw: &str) -> Option { let cost = parts.next().and_then(|value| value.parse().ok()).unwrap_or(1); Some(TenantRateLimit { rps, burst, cost: cost.max(1) }) } + +async fn list_tenant_rate_limit_rules(state: &AppState, filters: &HashMap) -> Result, ServiceError> { + let pattern = format!("{}*", state.cfg.tenant_rate_limit_prefix); + let mut stream = state.redis.scan_buffered(pattern, Some(100), None); + let mut out = Vec::new(); + + while let Some(key) = stream.next().await { + let key = key?.into_string().unwrap_or_default(); + if is_legacy_tenant_rate_limit_key(&key) { + continue; + } + + let raw: Option = state.redis.get(key.as_str()).await?; + let Some(limit) = raw.and_then(|raw| parse_tenant_rate_limit(&raw)) else { + continue; + }; + let Some(mut rule) = tenant_rate_limit_rule_from_key(state, &key, limit) else { + continue; + }; + rule.cost = rule.cost.max(1); + if tenant_rule_matches_filters(&rule, filters) { + out.push(TenantRateLimitRuleView { key, rule }); + } + } + + out.sort_by(|a, b| tenant_rule_specificity(&b.rule).cmp(&tenant_rule_specificity(&a.rule)).then_with(|| a.key.cmp(&b.key))); + Ok(out) +} + +async fn upsert_tenant_rate_limit_rule(state: &AppState, mut rule: TenantRateLimitRule) -> Result { + validate_tenant_rate_limit_rule(&rule)?; + rule.cost = rule.cost.max(1); + let key = tenant_rate_limit_rule_key(state, &rule); + let value = serde_json::to_string(&TenantRateLimit { + rps: rule.rps, + burst: rule.burst, + cost: rule.cost, + }) + .map_err(|e| ServiceError::internal(format!("serialize tenant rate limit: {e}")))?; + let _: String = state.redis.set(key.as_str(), value, None, None, false).await?; + // TODO(v2): if rule.ttl_secs is Some, apply Redis EXPIRE/PEXPIRE here. + Ok(TenantRateLimitRuleView { key, rule }) +} + +async fn delete_tenant_rate_limit_rule(state: &AppState, rule: TenantRateLimitRule) -> Result { + validate_tenant_rule_dimensions(&rule)?; + let key = tenant_rate_limit_rule_key(state, &rule); + let removed: u64 = state.redis.del(key.as_str()).await?; + Ok(removed) +} + +fn tenant_rate_limit_rule_key(state: &AppState, rule: &TenantRateLimitRule) -> String { + let base = format!("{}{}", state.cfg.tenant_rate_limit_prefix, sanitize_key(rule.tenant.trim())); + let mut key = base; + if let Some(model) = non_empty_opt(&rule.model) { + key.push_str(":model:"); + key.push_str(&sanitize_key(model)); + } + if let Some(path) = non_empty_opt(&rule.path) { + key.push_str(":path:"); + key.push_str(&sanitize_key(path)); + } + if let Some(policy) = non_empty_opt(&rule.policy) { + key.push_str(":policy:"); + key.push_str(&sanitize_key(policy)); + } + key +} + +fn tenant_rate_limit_rule_from_key(state: &AppState, key: &str, limit: TenantRateLimit) -> Option { + let rest = key.strip_prefix(&state.cfg.tenant_rate_limit_prefix)?; + let mut parts = rest.split(':'); + let tenant = parts.next()?.to_string(); + if tenant.is_empty() { + return None; + } + + let mut model = None; + let mut path = None; + let mut policy = None; + while let (Some(name), Some(value)) = (parts.next(), parts.next()) { + match name { + "model" => model = Some(value.to_string()), + "path" => path = Some(value.to_string()), + "policy" => policy = Some(value.to_string()), + _ => {} + } + } + + Some(TenantRateLimitRule { + tenant, + model, + path, + policy, + rps: limit.rps, + burst: limit.burst, + cost: limit.cost.max(1), + ttl_secs: None, + }) +} + +fn validate_tenant_rate_limit_rule(rule: &TenantRateLimitRule) -> Result<(), ServiceError> { + validate_tenant_rule_dimensions(rule)?; + if rule.rps == 0 { + return Err(ServiceError::bad_request("rps must be greater than 0")); + } + if rule.burst == 0 { + return Err(ServiceError::bad_request("burst must be greater than 0")); + } + if rule.cost == 0 { + return Err(ServiceError::bad_request("cost must be greater than 0")); + } + Ok(()) +} + +fn validate_tenant_rule_dimensions(rule: &TenantRateLimitRule) -> Result<(), ServiceError> { + if rule.tenant.trim().is_empty() { + return Err(ServiceError::bad_request("tenant is required")); + } + if let Some(policy) = non_empty_opt(&rule.policy) { + match policy { + "abandon" | "queue" | "wait" => {} + _ => return Err(ServiceError::bad_request("policy must be abandon, queue, or wait")), + } + } + Ok(()) +} + +fn tenant_rule_matches_filters(rule: &TenantRateLimitRule, filters: &HashMap) -> bool { + for (name, value) in filters { + let value = value.trim(); + if value.is_empty() { + continue; + } + let matches = match name.as_str() { + "tenant" => rule.tenant.contains(value), + "model" => rule.model.as_deref().unwrap_or("").contains(value), + "path" => rule.path.as_deref().unwrap_or("").contains(value), + "policy" => rule.policy.as_deref().unwrap_or("") == value, + _ => true, + }; + if !matches { + return false; + } + } + true +} + +fn tenant_rule_specificity(rule: &TenantRateLimitRule) -> usize { + usize::from(non_empty_opt(&rule.model).is_some()) + usize::from(non_empty_opt(&rule.path).is_some()) + usize::from(non_empty_opt(&rule.policy).is_some()) +} + +fn is_legacy_tenant_rate_limit_key(key: &str) -> bool { + key.ends_with(":rps") || key.ends_with(":burst") || key.ends_with(":cost") +} + +fn non_empty_opt(value: &Option) -> Option<&str> { + value.as_deref().map(str::trim).filter(|value| !value.is_empty()) +} diff --git a/binary/ai-gateway-service/src/app/runtime.rs b/binary/ai-gateway-service/src/app/runtime.rs index cdee8851..6a3bfc23 100644 --- a/binary/ai-gateway-service/src/app/runtime.rs +++ b/binary/ai-gateway-service/src/app/runtime.rs @@ -28,7 +28,12 @@ pub async fn run() -> Result<(), Box> { .route("/v1/queue/enqueue", post(enqueue)) .route("/v1/queue/enqueue-and-wait", post(enqueue_and_wait)) .route("/v1/jobs/{job_id}", get(get_job)) + .route("/v1/admin/plugins/{plugin}/schema", get(admin_plugin_schema)) + .route("/v1/admin/plugins/{plugin}/readme", get(admin_plugin_readme)) + .route("/v1/admin/tenant-rate-limits", get(admin_list_tenant_rate_limits).put(admin_upsert_tenant_rate_limit).delete(admin_delete_tenant_rate_limit)) .layer(DefaultBodyLimit::max(args.max_body_bytes)) + // TODO(v2): replace permissive CORS with authenticated admin ingress once the UI is proxied through the admin backend. + .layer(CorsLayer::permissive()) .layer(TraceLayer::new_for_http()) .with_state(state); diff --git a/binary/ai-gateway-service/src/app/types.rs b/binary/ai-gateway-service/src/app/types.rs index b29ff26f..a3be7a4d 100644 --- a/binary/ai-gateway-service/src/app/types.rs +++ b/binary/ai-gateway-service/src/app/types.rs @@ -243,7 +243,7 @@ impl QueuePriority { } } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema)] struct TenantRateLimit { rps: u64, burst: u64, @@ -251,10 +251,222 @@ struct TenantRateLimit { cost: u64, } +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +struct AiGatewayQueuePluginConfig { + #[serde(default)] + service: AiGatewayServiceConfig, + #[serde(default)] + paths: AiGatewayPathsConfig, + #[serde(default)] + headers: AiGatewayHeadersConfig, + #[serde(default)] + policies: AiGatewayPoliciesConfig, + #[serde(default)] + priority: AiGatewayPriorityConfig, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +struct AiGatewayServiceConfig { + #[serde(default = "default_service_cluster")] + cluster: String, + #[serde(default = "default_service_authority")] + authority: String, + #[serde(default = "default_service_timeout_ms")] + timeout_ms: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +struct AiGatewayPathsConfig { + #[serde(default = "default_rate_limit_path")] + rate_limit: String, + #[serde(default = "default_enqueue_path")] + enqueue: String, + #[serde(default = "default_wait_path")] + wait: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +struct AiGatewayHeadersConfig { + #[serde(default = "default_policy_header")] + policy: String, + #[serde(default = "default_tenant_header")] + tenant: String, + #[serde(default = "default_model_header")] + model: String, + #[serde(default = "default_priority_header")] + priority: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +struct AiGatewayPoliciesConfig { + #[serde(default = "default_require_policy")] + require: bool, + #[serde(default)] + default: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +struct AiGatewayPriorityConfig { + #[serde(default = "default_priority_enabled")] + enabled: bool, + #[serde(default = "default_priority")] + default: String, + #[serde(default)] + high_models: Vec, + #[serde(default)] + low_models: Vec, + #[serde(default)] + high_tenants: Vec, + #[serde(default)] + low_tenants: Vec, +} + +impl Default for AiGatewayQueuePluginConfig { + fn default() -> Self { + Self { + service: AiGatewayServiceConfig::default(), + paths: AiGatewayPathsConfig::default(), + headers: AiGatewayHeadersConfig::default(), + policies: AiGatewayPoliciesConfig::default(), + priority: AiGatewayPriorityConfig::default(), + } + } +} + +impl Default for AiGatewayServiceConfig { + fn default() -> Self { + Self { + cluster: default_service_cluster(), + authority: default_service_authority(), + timeout_ms: default_service_timeout_ms(), + } + } +} + +impl Default for AiGatewayPathsConfig { + fn default() -> Self { + Self { + rate_limit: default_rate_limit_path(), + enqueue: default_enqueue_path(), + wait: default_wait_path(), + } + } +} + +impl Default for AiGatewayHeadersConfig { + fn default() -> Self { + Self { + policy: default_policy_header(), + tenant: default_tenant_header(), + model: default_model_header(), + priority: default_priority_header(), + } + } +} + +impl Default for AiGatewayPoliciesConfig { + fn default() -> Self { + Self { + require: default_require_policy(), + default: None, + } + } +} + +impl Default for AiGatewayPriorityConfig { + fn default() -> Self { + Self { + enabled: default_priority_enabled(), + default: default_priority(), + high_models: Vec::new(), + low_models: Vec::new(), + high_tenants: Vec::new(), + low_tenants: Vec::new(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +struct TenantRateLimitRule { + tenant: String, + #[serde(default)] + model: Option, + #[serde(default)] + path: Option, + #[serde(default)] + policy: Option, + rps: u64, + burst: u64, + #[serde(default = "default_rate_limit_cost")] + cost: u64, + // TODO(v2): apply ttl_secs to the Redis key via EXPIRE/PEXPIRE when time-bound rules are enabled. + #[serde(default)] + ttl_secs: Option, +} + +#[derive(Debug, Clone, Serialize)] +struct TenantRateLimitRuleView { + key: String, + #[serde(flatten)] + rule: TenantRateLimitRule, +} + fn default_rate_limit_cost() -> u64 { 1 } +fn default_service_cluster() -> String { + "ai-gateway-service".to_string() +} + +fn default_service_authority() -> String { + "ai-gateway-service".to_string() +} + +fn default_service_timeout_ms() -> u64 { + 65_000 +} + +fn default_rate_limit_path() -> String { + "/v1/ratelimit/check".to_string() +} + +fn default_enqueue_path() -> String { + "/v1/queue/enqueue".to_string() +} + +fn default_wait_path() -> String { + "/v1/queue/enqueue-and-wait".to_string() +} + +fn default_policy_header() -> String { + "x-ratelimit-policy".to_string() +} + +fn default_tenant_header() -> String { + "x-tenant-id".to_string() +} + +fn default_model_header() -> String { + "x-model".to_string() +} + +fn default_priority_header() -> String { + "x-queue-priority".to_string() +} + +fn default_require_policy() -> bool { + true +} + +fn default_priority_enabled() -> bool { + true +} + +fn default_priority() -> String { + "normal".to_string() +} + impl QueuePolicy { fn as_str(self) -> &'static str { match self { diff --git a/plugins/wasm/ai-gateway-queue/README.md b/plugins/wasm/ai-gateway-queue/README.md index 50bdd208..c6f8767a 100644 --- a/plugins/wasm/ai-gateway-queue/README.md +++ b/plugins/wasm/ai-gateway-queue/README.md @@ -1,12 +1,14 @@ # ai-gateway-queue -`ai-gateway-queue` 是一个运行在 SpaceGate Wasm 里的 AI 网关插件，用来把请求按策略分成三种处理方式： +`ai-gateway-queue` 是一个运行在 SpaceGate Wasm 里的 **AI 请求队列网关**插件：在入口处对 AI 请求按租户做准入判定（基于令牌桶速率），命中后按选定的队列模式把请求分流到 Redis 多优先级队列异步消化，配合回调重试和对象存储 offload 实现无损交付。 -- `abandon`：先做限流检查，失败直接返回 -- `queue`：进入外部队列，立即返回 `202` -- `wait`：进入外部队列并等待结果返回 +支持三种队列模式（通过 `X-RateLimit-Policy` 请求头选择，名字保留兼容历史）： -它本身不直接访问 Redis，而是通过 `dispatch_http_call` 调用外部的 `ai-gateway-service`，再由该服务去处理 Redis、队列和等待逻辑。 +- `abandon`：超额请求直接返回 429（不入队，等价于纯节流闸门） +- `queue`：超额请求入队后立即返回 `202`，结果通过回调或轮询拿到 +- `wait`：超额请求入队后同步等待结果返回（类长轮询） + +插件本身不直接访问 Redis，而是通过 `dispatch_http_call` 调用外部队列后端（`ai-gateway-service`），再由该后端处理 Redis Streams、worker 消费、回调重试、结果回收等队列基础设施。 ## 架构 From 6f2d165ca5dac9c2136fc7ab3cd5f09ae8be6fb3 Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Fri, 22 May 2026 00:23:53 +0800 Subject: [PATCH 14/19] feat: complete AI queue gateway parity and e2e fixes Add tenant quota TTL expiry, aligned callback/status payloads, richer metrics, object-store body offload tests, Wasm VM pause handling, and demo gateway resources for SpaceGate smoke runs. Co-authored-by: Cursor --- .dockerignore | 3 + binary/ai-gateway-service/Cargo.toml | 2 +- binary/ai-gateway-service/README.md | 21 ++++ binary/ai-gateway-service/src/app.rs | 3 +- binary/ai-gateway-service/src/app/callback.rs | 28 ++--- binary/ai-gateway-service/src/app/handlers.rs | 31 +++++- binary/ai-gateway-service/src/app/metrics.rs | 45 ++++++-- binary/ai-gateway-service/src/app/queue.rs | 30 +++-- .../ai-gateway-service/src/app/ratelimit.rs | 60 +++++++--- binary/ai-gateway-service/src/app/runtime.rs | 24 +++- binary/ai-gateway-service/src/app/tests.rs | 90 ++++++++++++++- binary/ai-gateway-service/src/app/types.rs | 91 ++++++++++++++- binary/ai-gateway-service/src/app/util.rs | 105 ++++++++++++++++++ crates/plugin-wasm/src/vm.rs | 8 +- plugins/wasm/ai-gateway-queue/src/lib.rs | 6 +- resource/ai-gateway-demo/config.json | 5 + .../gateway/ai-demo/config.json | 23 ++++ .../gateway/ai-demo/route/ai.json | 33 ++++++ .../plugin/wasm.ai-gateway-queue.json | 2 +- 19 files changed, 548 insertions(+), 62 deletions(-) create mode 100644 .dockerignore create mode 100644 resource/ai-gateway-demo/config.json create mode 100644 resource/ai-gateway-demo/gateway/ai-demo/config.json create mode 100644 resource/ai-gateway-demo/gateway/ai-demo/route/ai.json diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..568383a0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +target +.git +**/*.md diff --git a/binary/ai-gateway-service/Cargo.toml b/binary/ai-gateway-service/Cargo.toml index dc316b41..49efd520 100644 --- a/binary/ai-gateway-service/Cargo.toml +++ b/binary/ai-gateway-service/Cargo.toml @@ -18,7 +18,7 @@ base64 = { workspace = true } bytes = { workspace = true } clap = { version = "4.5", features = ["derive", "env"] } futures-util = { workspace = true } -fred = { version = "10.1.0", default-features = false, features = ["enable-rustls", "i-keys", "i-scripts", "i-streams", "subscriber-client", "transactions"] } +fred = { version = "10.1.0", default-features = false, features = ["default-nil-types", "enable-rustls", "i-keys", "i-scripts", "i-streams", "subscriber-client", "transactions"] } http = "1" reqwest = { workspace = true, features = ["json"] } schemars = "0.8" diff --git a/binary/ai-gateway-service/README.md b/binary/ai-gateway-service/README.md index f7e94c4d..b1da5609 100644 --- a/binary/ai-gateway-service/README.md +++ b/binary/ai-gateway-service/README.md @@ -132,3 +132,24 @@ For job processing, each entry acquires a Redis lease key before upstream execut - `job_dlq_depth` and `callback_dlq_depth` for exhausted jobs and callbacks. - `enqueue_latency_ms_*`, `enqueue_body_size_bytes_*`, `wait_total`, and `wait_timeout_total` for ingress and wait-mode health. - `worker_processing_time_ms_*`, `worker_completed_total`, `worker_failed_total`, `reclaimed_total`, `lease_skip_total`, and `job_dlq_total` for worker health. +- `object_offload_total` and `object_multipart_abort_total` for large-body offload. + +## Body offload tests + +Unit tests (mock S3 multipart server, no Docker): + +```bash +cargo test -p ai-gateway-service store_body_ +``` + +MinIO end-to-end (Docker + worker roundtrip): + +```bash +# 需要：Redis、mock 上游 :9000、Docker +./tests/queue-object-store-e2e.sh +``` + +The script starts MinIO on `:9001` by default (avoids clashing with the mock upstream on `:9000`), launches a dedicated `ai-gateway-service` on `:18081` with `AI_OBJECT_STORE_ENDPOINT`, and verifies: + +- inline body below `AI_INLINE_THRESHOLD` does not increment `object_offload_total` +- larger body is stored in MinIO and the worker completes after `load_body()` fetches it diff --git a/binary/ai-gateway-service/src/app.rs b/binary/ai-gateway-service/src/app.rs index 6aab3504..2b934515 100644 --- a/binary/ai-gateway-service/src/app.rs +++ b/binary/ai-gateway-service/src/app.rs @@ -19,8 +19,9 @@ use fred::types::ExpireOptions; use futures_util::StreamExt; use schemars::{schema_for, JsonSchema}; use serde::{Deserialize, Serialize}; +use std::sync::Mutex; use tokio::sync::Semaphore; -use tower_http::cors::CorsLayer; +use tower_http::cors::{Any, CorsLayer}; use tower_http::trace::TraceLayer; include!("app/types.rs"); diff --git a/binary/ai-gateway-service/src/app/callback.rs b/binary/ai-gateway-service/src/app/callback.rs index e7ed0de2..dd3cbcc9 100644 --- a/binary/ai-gateway-service/src/app/callback.rs +++ b/binary/ai-gateway-service/src/app/callback.rs @@ -4,8 +4,9 @@ fn callback_body(result: &StoredResult) -> serde_json::Value { "status": result.status, "http_status": result.http_status, "headers": result.headers, + "result": decode_callback_result(&result.body_base64), "body_base64": result.body_base64, - "result": result.body_base64, + "completed_at": format_completed_at_rfc3339(result.completed_at_ms), "completed_at_ms": result.completed_at_ms, "error": result.error, }) @@ -75,18 +76,17 @@ fn spawn_callback_retry_worker(state: AppState) { async fn callback_retry_once(state: &AppState) -> Result<(), ServiceError> { reclaim_callback_retries(state).await?; - let reply: XReadResponse = state - .redis - .xreadgroup_map( - state.cfg.callback_retry_group.as_str(), - state.cfg.consumer_name.as_str(), - Some(5), - Some(1000), - false, - vec![state.cfg.callback_retry_stream.as_str()], - vec![">"], - ) - .await?; + let reply = xreadgroup_map_or_empty( + &state.worker_redis, + state.cfg.callback_retry_group.as_str(), + state.cfg.consumer_name.as_str(), + Some(5), + Some(1000), + false, + vec![state.cfg.callback_retry_stream.as_str()], + vec![">"], + ) + .await?; for (_stream, entries) in reply { for (entry_id, fields) in entries { @@ -100,7 +100,7 @@ async fn reclaim_callback_retries(state: &AppState) -> Result<(), ServiceError> let consumer = format!("{}-callback-reclaimer", state.cfg.consumer_name); let min_idle_ms = state.cfg.callback_retry_reclaim_idle_secs.saturating_mul(1000); let (_cursor, entries): (String, Vec<(String, HashMap)>) = state - .redis + .worker_redis .xautoclaim_values( state.cfg.callback_retry_stream.as_str(), state.cfg.callback_retry_group.as_str(), diff --git a/binary/ai-gateway-service/src/app/handlers.rs b/binary/ai-gateway-service/src/app/handlers.rs index 208bc797..0bd4f69e 100644 --- a/binary/ai-gateway-service/src/app/handlers.rs +++ b/binary/ai-gateway-service/src/app/handlers.rs @@ -25,6 +25,14 @@ async fn check_rate_limit(State(state): State, headers: HeaderMap, uri let allowed = out.first().copied().unwrap_or(0) == 1; if !allowed { state.metrics.rate_limited_total.fetch_add(1, Ordering::Relaxed); + inc_labeled( + &state.metrics, + format!( + r#"rate_limited_total{{policy="{}",tenant="{}"}}"#, + metrics_label(&policy), + metrics_label(&tenant) + ), + ); } Ok(Json(RateLimitResponse { allowed, @@ -112,6 +120,22 @@ async fn metrics(State(state): State) -> Result 0 { + wait_timeout_total as f64 / wait_total as f64 + } else { + 0.0 + }; + let callback_failure_rate = if worker_completed_total > 0 { + callback_failure_total as f64 / worker_completed_total as f64 + } else { + 0.0 + }; + let labeled_lines = format_labeled_lines(&state.metrics); + let body = format!( "\ rate_limited_total {}\n\ @@ -135,7 +159,9 @@ enqueue_body_size_bytes_bucket{{le=\"5242880\"}} {}\n\ enqueue_body_size_bytes_bucket{{le=\"+Inf\"}} {}\n\ wait_total {}\n\ wait_timeout_total {}\n\ +wait_timeout_rate {:.6}\n\ callback_failure_total {}\n\ +callback_failure_rate {:.6}\n\ callback_retry_total {}\n\ callback_retry_success_total {}\n\ callback_retry_dlq_total {}\n\ @@ -163,7 +189,8 @@ pel_size{{priority=\"low\"}} {}\n\ job_dlq_depth {}\n\ callback_retry_depth {}\n\ callback_retry_pel_size {}\n\ -callback_dlq_depth {}\n", +callback_dlq_depth {}\n\ +{labeled_lines}\n", state.metrics.rate_limited_total.load(Ordering::Relaxed), state.metrics.enqueue_total.load(Ordering::Relaxed), state.metrics.enqueue_queue_total.load(Ordering::Relaxed), @@ -187,7 +214,9 @@ callback_dlq_depth {}\n", state.metrics.body_size_count.load(Ordering::Relaxed), state.metrics.wait_total.load(Ordering::Relaxed), state.metrics.wait_timeout_total.load(Ordering::Relaxed), + wait_timeout_rate, state.metrics.callback_failure_total.load(Ordering::Relaxed), + callback_failure_rate, state.metrics.callback_retry_total.load(Ordering::Relaxed), state.metrics.callback_retry_success_total.load(Ordering::Relaxed), state.metrics.callback_retry_dlq_total.load(Ordering::Relaxed), diff --git a/binary/ai-gateway-service/src/app/metrics.rs b/binary/ai-gateway-service/src/app/metrics.rs index 887866e1..18ed9f12 100644 --- a/binary/ai-gateway-service/src/app/metrics.rs +++ b/binary/ai-gateway-service/src/app/metrics.rs @@ -41,18 +41,41 @@ fn pending_count_from_value(value: &Value) -> i64 { } } -fn observe_enqueue_latency(metrics: &Metrics, elapsed_ms: u64) { +fn inc_labeled(metrics: &Metrics, key: impl Into) { + let mut map = metrics.labeled.lock().unwrap_or_else(|error| error.into_inner()); + *map.entry(key.into()).or_insert(0) += 1; +} + +fn format_labeled_lines(metrics: &Metrics) -> String { + let map = metrics.labeled.lock().unwrap_or_else(|error| error.into_inner()); + let mut keys: Vec<_> = map.keys().cloned().collect(); + keys.sort(); + keys.into_iter() + .filter_map(|key| map.get(&key).copied().map(|value| format!("{key} {value}"))) + .collect::>() + .join("\n") +} + +fn observe_enqueue_latency(metrics: &Metrics, elapsed_ms: u64, policy: &str, size_bucket: &str) { metrics.enqueue_latency_count.fetch_add(1, Ordering::Relaxed); metrics.enqueue_latency_sum_ms.fetch_add(elapsed_ms, Ordering::Relaxed); - if elapsed_ms <= 100 { + let le = if elapsed_ms <= 100 { metrics.enqueue_latency_le_100_ms.fetch_add(1, Ordering::Relaxed); + "100" } else if elapsed_ms <= 500 { metrics.enqueue_latency_le_500_ms.fetch_add(1, Ordering::Relaxed); + "500" } else if elapsed_ms <= 1000 { metrics.enqueue_latency_le_1000_ms.fetch_add(1, Ordering::Relaxed); + "1000" } else { metrics.enqueue_latency_gt_1000_ms.fetch_add(1, Ordering::Relaxed); - } + "+Inf" + }; + inc_labeled( + metrics, + format!(r#"enqueue_latency_ms_bucket{{policy="{policy}",size_bucket="{size_bucket}",le="{le}"}}"#), + ); } fn observe_body_size(metrics: &Metrics, size: usize) { @@ -69,17 +92,25 @@ fn observe_body_size(metrics: &Metrics, size: usize) { } } -fn observe_worker_processing(metrics: &Metrics, elapsed_ms: u64) { +fn observe_worker_processing(metrics: &Metrics, elapsed_ms: u64, model: &str) { metrics.worker_processing_count.fetch_add(1, Ordering::Relaxed); metrics.worker_processing_sum_ms.fetch_add(elapsed_ms, Ordering::Relaxed); - if elapsed_ms <= 1000 { + let model = metrics_label(model); + let le = if elapsed_ms <= 1000 { metrics.worker_processing_le_1000_ms.fetch_add(1, Ordering::Relaxed); + "1000" } else if elapsed_ms <= 5000 { metrics.worker_processing_le_5000_ms.fetch_add(1, Ordering::Relaxed); + "5000" } else if elapsed_ms <= 30_000 { metrics.worker_processing_le_30000_ms.fetch_add(1, Ordering::Relaxed); + "30000" } else { metrics.worker_processing_gt_30000_ms.fetch_add(1, Ordering::Relaxed); - } + "+Inf" + }; + inc_labeled( + metrics, + format!(r#"worker_processing_time_ms_bucket{{model="{model}",le="{le}"}}"#), + ); } - diff --git a/binary/ai-gateway-service/src/app/queue.rs b/binary/ai-gateway-service/src/app/queue.rs index 2c2b378d..75ecd967 100644 --- a/binary/ai-gateway-service/src/app/queue.rs +++ b/binary/ai-gateway-service/src/app/queue.rs @@ -41,7 +41,12 @@ async fn enqueue_job(state: &AppState, policy: QueuePolicy, _method: Method, uri trim_stream(state, &stream_key).await?; state.metrics.enqueue_total.fetch_add(1, Ordering::Relaxed); - observe_enqueue_latency(&state.metrics, now_ms().saturating_sub(enqueue_started_at)); + observe_enqueue_latency( + &state.metrics, + now_ms().saturating_sub(enqueue_started_at), + policy.as_str(), + body_size_bucket(body_ref.size, body_ref.storage), + ); observe_body_size(&state.metrics, body_ref.size); match priority { QueuePriority::High => { @@ -69,7 +74,8 @@ async fn enqueue_job(state: &AppState, policy: QueuePolicy, _method: Method, uri stream_id, stream_key, status: "queued", - poll_url: format!("/v1/jobs/{job_id}"), + poll_url: job_poll_url(&job_id), + status_url: job_status_url_legacy(&job_id), }, created_at_ms: created_at, }) @@ -103,8 +109,17 @@ async fn worker_once(state: &AppState, consumer: &str) -> Result<(), ServiceErro } async fn read_worker_stream(state: &AppState, consumer: &str, stream: &str, block_ms: u64) -> Result { - let reply: XReadResponse = - state.redis.xreadgroup_map(state.cfg.consumer_group.as_str(), consumer, Some(5), Some(block_ms), false, vec![stream], vec![">"]).await?; + let reply = xreadgroup_map_or_empty( + &state.worker_redis, + state.cfg.consumer_group.as_str(), + consumer, + Some(5), + Some(block_ms), + false, + vec![stream], + vec![">"], + ) + .await?; let mut processed = 0; for (_stream, entries) in reply { @@ -144,16 +159,17 @@ async fn process_stream_entry(state: &AppState, stream: &str, entry_id: &str, fi } let processing_started_at = now_ms(); + let model = field_string(fields, "model").unwrap_or_else(|| "default".to_string()); match process_job(state, stream, entry_id, fields).await { Ok(()) => { - observe_worker_processing(&state.metrics, now_ms().saturating_sub(processing_started_at)); + observe_worker_processing(&state.metrics, now_ms().saturating_sub(processing_started_at), &model); ack_stream_entry(state, stream, entry_id).await?; clear_job_delivery_attempt(state, &job_id).await; release_job_lease(state, &job_id).await; Ok(true) } Err(e) => { - observe_worker_processing(&state.metrics, now_ms().saturating_sub(processing_started_at)); + observe_worker_processing(&state.metrics, now_ms().saturating_sub(processing_started_at), &model); release_job_lease(state, &job_id).await; Err(e) } @@ -322,7 +338,7 @@ async fn reclaim_once(state: &AppState) -> Result<(), ServiceError> { let min_idle_ms = state.cfg.reclaim_min_idle_secs.saturating_mul(1000); for stream in configured_streams(state) { let (_cursor, entries): (String, Vec<(String, HashMap)>) = - state.redis.xautoclaim_values(stream.as_str(), state.cfg.consumer_group.as_str(), consumer.as_str(), min_idle_ms, "0-0", Some(10), false).await?; + state.worker_redis.xautoclaim_values(stream.as_str(), state.cfg.consumer_group.as_str(), consumer.as_str(), min_idle_ms, "0-0", Some(10), false).await?; for (entry_id, fields) in entries { match process_stream_entry(state, stream.as_str(), entry_id.as_str(), &fields).await { Ok(true) => { diff --git a/binary/ai-gateway-service/src/app/ratelimit.rs b/binary/ai-gateway-service/src/app/ratelimit.rs index 6da10542..89869643 100644 --- a/binary/ai-gateway-service/src/app/ratelimit.rs +++ b/binary/ai-gateway-service/src/app/ratelimit.rs @@ -1,7 +1,7 @@ async fn tenant_rate_limit(state: &AppState, tenant: &str, model: &str, path: &str, policy: &str) -> Result { for key in tenant_rate_limit_candidate_keys(state, tenant, model, path, policy) { let raw: Option = state.redis.get(key.as_str()).await.unwrap_or(None); - if let Some(limit) = raw.and_then(|raw| parse_tenant_rate_limit(&raw)) { + if let Some(limit) = raw.and_then(|raw| parse_stored_tenant_rate_limit(&raw).map(|stored| stored.limit)) { return Ok(limit); } } @@ -34,16 +34,38 @@ fn tenant_rate_limit_candidate_keys(state: &AppState, tenant: &str, model: &str, ] } -fn parse_tenant_rate_limit(raw: &str) -> Option { +struct ParsedStoredTenantRateLimit { + limit: TenantRateLimit, + ttl_secs: Option, +} + +fn parse_stored_tenant_rate_limit(raw: &str) -> Option { let raw = raw.trim(); if raw.is_empty() { return None; } + if let Ok(stored) = serde_json::from_str::(raw) { + return Some(ParsedStoredTenantRateLimit { + limit: TenantRateLimit { + rps: stored.rps, + burst: stored.burst, + cost: stored.cost.max(1), + }, + ttl_secs: stored.ttl_secs, + }); + } if let Ok(mut limit) = serde_json::from_str::(raw) { limit.cost = limit.cost.max(1); - return Some(limit); + return Some(ParsedStoredTenantRateLimit { limit, ttl_secs: None }); } + parse_tenant_rate_limit_csv(raw).map(|limit| ParsedStoredTenantRateLimit { limit, ttl_secs: None }) +} +fn parse_tenant_rate_limit(raw: &str) -> Option { + parse_stored_tenant_rate_limit(raw).map(|stored| stored.limit) +} + +fn parse_tenant_rate_limit_csv(raw: &str) -> Option { let mut parts = raw.split(',').map(str::trim); let rps = parts.next()?.parse().ok()?; let burst = parts.next()?.parse().ok()?; @@ -63,19 +85,20 @@ async fn list_tenant_rate_limit_rules(state: &AppState, filters: &HashMap = state.redis.get(key.as_str()).await?; - let Some(limit) = raw.and_then(|raw| parse_tenant_rate_limit(&raw)) else { + let Some(stored) = raw.and_then(|raw| parse_stored_tenant_rate_limit(&raw)) else { continue; }; - let Some(mut rule) = tenant_rate_limit_rule_from_key(state, &key, limit) else { + let Some(mut rule) = tenant_rate_limit_rule_from_key(state, &key, stored.limit, stored.ttl_secs) else { continue; }; rule.cost = rule.cost.max(1); if tenant_rule_matches_filters(&rule, filters) { - out.push(TenantRateLimitRuleView { key, rule }); + let ttl_remaining_secs = read_ttl_remaining_secs(state, key.as_str()).await; + out.push(tenant_rate_limit_rule_view(key, rule, ttl_remaining_secs)); } } - out.sort_by(|a, b| tenant_rule_specificity(&b.rule).cmp(&tenant_rule_specificity(&a.rule)).then_with(|| a.key.cmp(&b.key))); + out.sort_by(|a, b| tenant_rule_specificity_rule(a).cmp(&tenant_rule_specificity_rule(b)).then_with(|| a.key.cmp(&b.key))); Ok(out) } @@ -83,15 +106,17 @@ async fn upsert_tenant_rate_limit_rule(state: &AppState, mut rule: TenantRateLim validate_tenant_rate_limit_rule(&rule)?; rule.cost = rule.cost.max(1); let key = tenant_rate_limit_rule_key(state, &rule); - let value = serde_json::to_string(&TenantRateLimit { + let value = serde_json::to_string(&StoredTenantRateLimit { rps: rule.rps, burst: rule.burst, cost: rule.cost, + ttl_secs: rule.ttl_secs, }) .map_err(|e| ServiceError::internal(format!("serialize tenant rate limit: {e}")))?; - let _: String = state.redis.set(key.as_str(), value, None, None, false).await?; - // TODO(v2): if rule.ttl_secs is Some, apply Redis EXPIRE/PEXPIRE here. - Ok(TenantRateLimitRuleView { key, rule }) + let expiration = rule.ttl_secs.map(|ttl| Expiration::EX(ttl.max(1) as i64)); + let _: String = state.redis.set(key.as_str(), value, expiration, None, false).await?; + let ttl_remaining_secs = read_ttl_remaining_secs(state, key.as_str()).await; + Ok(tenant_rate_limit_rule_view(key, rule, ttl_remaining_secs)) } async fn delete_tenant_rate_limit_rule(state: &AppState, rule: TenantRateLimitRule) -> Result { @@ -101,6 +126,11 @@ async fn delete_tenant_rate_limit_rule(state: &AppState, rule: TenantRateLimitRu Ok(removed) } +async fn read_ttl_remaining_secs(state: &AppState, key: &str) -> Option { + let ttl: i64 = state.redis.ttl(key).await.unwrap_or(-2); + if ttl > 0 { Some(ttl) } else { None } +} + fn tenant_rate_limit_rule_key(state: &AppState, rule: &TenantRateLimitRule) -> String { let base = format!("{}{}", state.cfg.tenant_rate_limit_prefix, sanitize_key(rule.tenant.trim())); let mut key = base; @@ -119,7 +149,7 @@ fn tenant_rate_limit_rule_key(state: &AppState, rule: &TenantRateLimitRule) -> S key } -fn tenant_rate_limit_rule_from_key(state: &AppState, key: &str, limit: TenantRateLimit) -> Option { +fn tenant_rate_limit_rule_from_key(state: &AppState, key: &str, limit: TenantRateLimit, ttl_secs: Option) -> Option { let rest = key.strip_prefix(&state.cfg.tenant_rate_limit_prefix)?; let mut parts = rest.split(':'); let tenant = parts.next()?.to_string(); @@ -147,7 +177,7 @@ fn tenant_rate_limit_rule_from_key(state: &AppState, key: &str, limit: TenantRat rps: limit.rps, burst: limit.burst, cost: limit.cost.max(1), - ttl_secs: None, + ttl_secs, }) } @@ -198,8 +228,8 @@ fn tenant_rule_matches_filters(rule: &TenantRateLimitRule, filters: &HashMap usize { - usize::from(non_empty_opt(&rule.model).is_some()) + usize::from(non_empty_opt(&rule.path).is_some()) + usize::from(non_empty_opt(&rule.policy).is_some()) +fn tenant_rule_specificity_rule(view: &TenantRateLimitRuleView) -> usize { + usize::from(non_empty_opt(&view.model).is_some()) + usize::from(non_empty_opt(&view.path).is_some()) + usize::from(non_empty_opt(&view.policy).is_some()) } fn is_legacy_tenant_rate_limit_key(key: &str) -> bool { diff --git a/binary/ai-gateway-service/src/app/runtime.rs b/binary/ai-gateway-service/src/app/runtime.rs index 6a3bfc23..2aabc39d 100644 --- a/binary/ai-gateway-service/src/app/runtime.rs +++ b/binary/ai-gateway-service/src/app/runtime.rs @@ -4,8 +4,11 @@ pub async fn run() -> Result<(), Box> { let args = Args::parse(); let redis = build_redis_client(&args.redis_url)?; let _redis_task = redis.init().await?; + let worker_redis = build_redis_client(&args.redis_url)?; + let _worker_redis_task = worker_redis.init().await?; let state = AppState { redis, + worker_redis, http: reqwest::Client::new(), cfg: Arc::new(args.clone()), body_permits: Arc::new(Semaphore::new(args.body_read_concurrency.max(1))), @@ -28,12 +31,12 @@ pub async fn run() -> Result<(), Box> { .route("/v1/queue/enqueue", post(enqueue)) .route("/v1/queue/enqueue-and-wait", post(enqueue_and_wait)) .route("/v1/jobs/{job_id}", get(get_job)) + .route("/jobs/{job_id}/status", get(get_job)) .route("/v1/admin/plugins/{plugin}/schema", get(admin_plugin_schema)) .route("/v1/admin/plugins/{plugin}/readme", get(admin_plugin_readme)) .route("/v1/admin/tenant-rate-limits", get(admin_list_tenant_rate_limits).put(admin_upsert_tenant_rate_limit).delete(admin_delete_tenant_rate_limit)) .layer(DefaultBodyLimit::max(args.max_body_bytes)) - // TODO(v2): replace permissive CORS with authenticated admin ingress once the UI is proxied through the admin backend. - .layer(CorsLayer::permissive()) + .layer(build_admin_cors_layer(&args)) .layer(TraceLayer::new_for_http()) .with_state(state); @@ -43,3 +46,20 @@ pub async fn run() -> Result<(), Box> { axum::serve(listener, app).await?; Ok(()) } + +fn build_admin_cors_layer(args: &Args) -> CorsLayer { + let origins: Vec = args + .admin_cors_origins + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .filter_map(|value| HeaderValue::from_str(value).ok()) + .collect(); + if origins.is_empty() { + return CorsLayer::permissive(); + } + CorsLayer::new() + .allow_origin(origins) + .allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE, Method::OPTIONS]) + .allow_headers(Any) +} diff --git a/binary/ai-gateway-service/src/app/tests.rs b/binary/ai-gateway-service/src/app/tests.rs index 421913a1..832dea4a 100644 --- a/binary/ai-gateway-service/src/app/tests.rs +++ b/binary/ai-gateway-service/src/app/tests.rs @@ -46,11 +46,11 @@ mod tests { #[test] fn observes_histogram_buckets_as_non_overlapping_counts() { let metrics = Metrics::default(); - observe_enqueue_latency(&metrics, 80); - observe_enqueue_latency(&metrics, 800); + observe_enqueue_latency(&metrics, 80, "queue", "inline"); + observe_enqueue_latency(&metrics, 800, "wait", "inline"); observe_body_size(&metrics, 8 * 1024); observe_body_size(&metrics, 256 * 1024); - observe_worker_processing(&metrics, 2000); + observe_worker_processing(&metrics, 2000, "gpt-4o-mini"); assert_eq!(metrics.enqueue_latency_count.load(Ordering::Relaxed), 2); assert_eq!(metrics.enqueue_latency_le_100_ms.load(Ordering::Relaxed), 1); @@ -82,4 +82,88 @@ mod tests { assert_eq!(parse_queue_priority("low"), Some(QueuePriority::Low)); assert_eq!(parse_queue_priority("urgent"), None); } + + fn test_app_state(object_store_endpoint: Option, inline_threshold: usize) -> AppState { + let mut args = Args::parse_from(["ai-gateway-service"]); + args.object_store_endpoint = object_store_endpoint; + args.inline_threshold = inline_threshold; + args.max_body_bytes = 8 * 1024 * 1024; + args.object_store_bucket = "ai-gateway-body".to_string(); + args.object_store_prefix = "bodies".to_string(); + args.object_multipart_part_size = 1024; + let redis = build_redis_client("redis://127.0.0.1/").expect("redis client"); + AppState { + redis: redis.clone(), + worker_redis: redis, + http: reqwest::Client::new(), + cfg: Arc::new(args), + body_permits: Arc::new(Semaphore::new(8)), + metrics: Arc::new(Metrics::default()), + } + } + + async fn mock_s3_handler(method: Method, uri: Uri, body: axum::body::Bytes, stored: Arc>>) -> Response { + let query = uri.query().unwrap_or(""); + if method == Method::POST && query == "uploads" { + return ( + StatusCode::OK, + [(http::header::CONTENT_TYPE, "application/xml")], + r#"test-upload"#, + ) + .into_response(); + } + if method == Method::PUT && query.contains("partNumber=") { + stored.lock().unwrap_or_else(|e| e.into_inner()).extend_from_slice(&body); + return (StatusCode::OK, [(http::header::ETAG, "\"part-etag\"")]).into_response(); + } + if method == Method::POST && query.contains("uploadId=") { + return StatusCode::OK.into_response(); + } + if method == Method::GET { + let bytes = stored.lock().unwrap_or_else(|e| e.into_inner()).clone(); + return (StatusCode::OK, bytes).into_response(); + } + StatusCode::NOT_FOUND.into_response() + } + + #[tokio::test] + async fn store_body_keeps_small_payload_inline() { + let state = test_app_state(None, 16 * 1024); + let payload = vec![1u8; 4096]; + let location = store_body(&state, "job-inline", Body::from(payload.clone())).await.expect("inline store"); + assert_eq!(location.storage, "inline"); + assert_eq!(location.size, payload.len()); + assert!(!location.body_base64.is_empty()); + assert_eq!(state.metrics.object_offload_total.load(Ordering::Relaxed), 0); + } + + #[tokio::test] + async fn store_body_offloads_large_payload_via_s3_multipart_and_load_body_roundtrips() { + let stored = Arc::new(std::sync::Mutex::new(Vec::new())); + let stored_for_handler = stored.clone(); + let app = Router::new().fallback(move |method: Method, uri: Uri, body: axum::body::Bytes| { + let stored_for_handler = stored_for_handler.clone(); + async move { mock_s3_handler(method, uri, body, stored_for_handler).await } + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind mock s3"); + let addr = listener.local_addr().expect("mock s3 addr"); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("mock s3 serve"); + }); + + let state = test_app_state(Some(format!("http://{addr}")), 1024); + let payload = vec![7u8; 5000]; + let location = store_body(&state, "job-offload", Body::from(payload.clone())).await.expect("offload store"); + assert_eq!(location.storage, "object"); + assert_eq!(location.size, payload.len()); + assert!(location.body_base64.is_empty()); + assert!(location.object_ref.contains("job-offload")); + assert_eq!(state.metrics.object_offload_total.load(Ordering::Relaxed), 1); + + let mut fields = HashMap::new(); + fields.insert("storage".to_string(), Value::String("object".into())); + fields.insert("ref".to_string(), Value::String(location.object_ref.into())); + let loaded = load_body(&state, &fields).await.expect("load offloaded body"); + assert_eq!(loaded, payload); + } } diff --git a/binary/ai-gateway-service/src/app/types.rs b/binary/ai-gateway-service/src/app/types.rs index a3be7a4d..93a5c99a 100644 --- a/binary/ai-gateway-service/src/app/types.rs +++ b/binary/ai-gateway-service/src/app/types.rs @@ -19,7 +19,8 @@ local last_ts = tonumber(redis.call('GET', ts_key) or now) local elapsed = math.max(0, now - last_ts) tokens = math.min(burst_milli, tokens + elapsed * rate) -local ttl = math.max(1000, math.ceil((burst_milli / rate) * 2)) +-- TTL 必须显著大于典型连续判定窗口；过短会导致 key 过期后每次都回到满 burst。 +local ttl = math.max(300000, math.ceil((burst_milli / rate) * 10)) if tokens >= cost_milli then tokens = tokens - cost_milli redis.call('SET', tokens_key, tokens, 'PX', ttl) @@ -104,8 +105,11 @@ struct Args { tenant_rate_limit_prefix: String, #[arg(long, env = "AI_WAIT_TIMEOUT_SECS", default_value_t = 60)] wait_timeout_secs: u64, - #[arg(long, env = "AI_WORKER_CONCURRENCY", default_value_t = 1)] + #[arg(long, env = "AI_WORKER_CONCURRENCY", default_value_t = 10)] worker_concurrency: usize, + /// 逗号分隔的 Admin UI CORS 来源；为空则保持 permissive（本地开发）。 + #[arg(long, env = "AI_ADMIN_CORS_ORIGINS", default_value = "")] + admin_cors_origins: String, #[arg(long, env = "AI_UPSTREAM_BASE_URL")] upstream_base_url: Option, #[arg(long, env = "AI_MAX_BODY_BYTES", default_value_t = 32 * 1024 * 1024)] @@ -138,14 +142,16 @@ struct Args { #[derive(Clone)] struct AppState { + /// 非阻塞 API 路径专用连接（准入、入队、metrics、admin）。 redis: FredClient, + /// worker / reclaimer / callback-retry 专用连接，避免 BLOCK 型 XREADGROUP 占满 API 连接。 + worker_redis: FredClient, http: reqwest::Client, cfg: Arc, body_permits: Arc, metrics: Arc, } -#[derive(Default)] struct Metrics { rate_limited_total: AtomicU64, enqueue_total: AtomicU64, @@ -185,6 +191,54 @@ struct Metrics { lease_skip_total: AtomicU64, object_offload_total: AtomicU64, object_multipart_abort_total: AtomicU64, + /// Prometheus 带 label 的 counter（policy/tenant/model/size_bucket 等）。 + labeled: Mutex>, +} + +impl Default for Metrics { + fn default() -> Self { + Self { + rate_limited_total: AtomicU64::new(0), + enqueue_total: AtomicU64::new(0), + enqueue_queue_total: AtomicU64::new(0), + enqueue_wait_total: AtomicU64::new(0), + enqueue_priority_high_total: AtomicU64::new(0), + enqueue_priority_normal_total: AtomicU64::new(0), + enqueue_priority_low_total: AtomicU64::new(0), + enqueue_latency_count: AtomicU64::new(0), + enqueue_latency_sum_ms: AtomicU64::new(0), + enqueue_latency_le_100_ms: AtomicU64::new(0), + enqueue_latency_le_500_ms: AtomicU64::new(0), + enqueue_latency_le_1000_ms: AtomicU64::new(0), + enqueue_latency_gt_1000_ms: AtomicU64::new(0), + body_size_le_10kb: AtomicU64::new(0), + body_size_le_128kb: AtomicU64::new(0), + body_size_le_5mb: AtomicU64::new(0), + body_size_gt_5mb: AtomicU64::new(0), + body_size_count: AtomicU64::new(0), + body_size_sum_bytes: AtomicU64::new(0), + wait_total: AtomicU64::new(0), + wait_timeout_total: AtomicU64::new(0), + callback_failure_total: AtomicU64::new(0), + callback_retry_total: AtomicU64::new(0), + callback_retry_success_total: AtomicU64::new(0), + callback_retry_dlq_total: AtomicU64::new(0), + worker_completed_total: AtomicU64::new(0), + worker_failed_total: AtomicU64::new(0), + worker_processing_count: AtomicU64::new(0), + worker_processing_sum_ms: AtomicU64::new(0), + worker_processing_le_1000_ms: AtomicU64::new(0), + worker_processing_le_5000_ms: AtomicU64::new(0), + worker_processing_le_30000_ms: AtomicU64::new(0), + worker_processing_gt_30000_ms: AtomicU64::new(0), + reclaimed_total: AtomicU64::new(0), + job_dlq_total: AtomicU64::new(0), + lease_skip_total: AtomicU64::new(0), + object_offload_total: AtomicU64::new(0), + object_multipart_abort_total: AtomicU64::new(0), + labeled: Mutex::new(HashMap::new()), + } + } } #[derive(Debug, Serialize)] @@ -200,7 +254,10 @@ struct EnqueueResponse { stream_id: String, stream_key: String, status: &'static str, + /// 设计文档 poll 路径：`/jobs/{id}/status` poll_url: String, + /// 兼容旧客户端：`/v1/jobs/{id}` + status_url: String, } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -399,7 +456,17 @@ struct TenantRateLimitRule { burst: u64, #[serde(default = "default_rate_limit_cost")] cost: u64, - // TODO(v2): apply ttl_secs to the Redis key via EXPIRE/PEXPIRE when time-bound rules are enabled. + /// 临时配额 TTL（秒）；写入 Redis 时对 key 设置 EX。 + #[serde(default)] + ttl_secs: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +struct StoredTenantRateLimit { + rps: u64, + burst: u64, + #[serde(default = "default_rate_limit_cost")] + cost: u64, #[serde(default)] ttl_secs: Option, } @@ -407,8 +474,20 @@ struct TenantRateLimitRule { #[derive(Debug, Clone, Serialize)] struct TenantRateLimitRuleView { key: String, - #[serde(flatten)] - rule: TenantRateLimitRule, + tenant: String, + #[serde(skip_serializing_if = "Option::is_none")] + model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + policy: Option, + rps: u64, + burst: u64, + cost: u64, + #[serde(skip_serializing_if = "Option::is_none")] + ttl_secs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + ttl_remaining_secs: Option, } fn default_rate_limit_cost() -> u64 { diff --git a/binary/ai-gateway-service/src/app/util.rs b/binary/ai-gateway-service/src/app/util.rs index b3f66564..f21ca4ba 100644 --- a/binary/ai-gateway-service/src/app/util.rs +++ b/binary/ai-gateway-service/src/app/util.rs @@ -89,3 +89,108 @@ fn field_u64(fields: &HashMap, key: &str) -> Option { fn field_u32(fields: &HashMap, key: &str) -> Option { field_u64(fields, key).and_then(|value| value.try_into().ok()) } + +fn job_poll_url(job_id: &str) -> String { + format!("/jobs/{job_id}/status") +} + +fn job_status_url_legacy(job_id: &str) -> String { + format!("/v1/jobs/{job_id}") +} + +fn metrics_label(value: &str) -> String { + sanitize_key(value).chars().take(64).collect() +} + +fn body_size_bucket(size: usize, storage: &str) -> &'static str { + if storage == "s3" { + "s3" + } else if size <= 10 * 1024 { + "inline_small" + } else if size <= 128 * 1024 { + "inline" + } else { + "inline_large" + } +} + +fn format_completed_at_rfc3339(ms: u64) -> String { + let days = (ms / 86_400_000) as i64; + let rem_ms = ms % 86_400_000; + let (year, month, day) = civil_from_days(days); + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}.{:03}Z", + rem_ms / 3_600_000, + (rem_ms % 3_600_000) / 60_000, + (rem_ms % 60_000) / 1_000, + rem_ms % 1_000, + ) +} + +fn civil_from_days(z: i64) -> (i64, u32, u32) { + let z = z + 719468; + let era = if z >= 0 { z } else { z - 146096 } / 146097; + let doe = (z - era * 146097) as u64; + let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + let mut y = yoe as i64 + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + if m <= 2 { + y += 1; + } + (y, m as u32, d as u32) +} + +fn decode_callback_result(body_base64: &str) -> serde_json::Value { + if body_base64.is_empty() { + return serde_json::Value::Null; + } + let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(body_base64) else { + return serde_json::json!({ "raw_base64": body_base64 }); + }; + if let Ok(value) = serde_json::from_slice::(&bytes) { + return value; + } + serde_json::json!({ "raw_base64": body_base64 }) +} + +fn tenant_rate_limit_rule_view(key: String, rule: TenantRateLimitRule, ttl_remaining_secs: Option) -> TenantRateLimitRuleView { + TenantRateLimitRuleView { + key, + tenant: rule.tenant, + model: rule.model, + path: rule.path, + policy: rule.policy, + rps: rule.rps, + burst: rule.burst, + cost: rule.cost, + ttl_secs: rule.ttl_secs, + ttl_remaining_secs, + } +} + +/// Parse `XREADGROUP` into the map form used by workers. +/// +/// Redis returns `nil` when a blocking read times out or the stream has no new entries for `>`. +/// Without explicit handling, fred fails to convert that into `HashMap` and the worker aborts +/// before polling the next priority stream — even when another stream already has backlog. +async fn xreadgroup_map_or_empty( + redis: &FredClient, + group: &str, + consumer: &str, + count: Option, + block: Option, + noack: bool, + keys: Vec<&str>, + ids: Vec<&str>, +) -> Result, ServiceError> { + let value: Value = redis.xreadgroup(group, consumer, count, block, noack, keys, ids).await?; + if value.is_null() { + return Ok(HashMap::new()); + } + value + .into_xread_response() + .map_err(|e| ServiceError::internal(format!("parse xreadgroup response: {e}"))) +} diff --git a/crates/plugin-wasm/src/vm.rs b/crates/plugin-wasm/src/vm.rs index f9db0875..02aba8a9 100644 --- a/crates/plugin-wasm/src/vm.rs +++ b/crates/plugin-wasm/src/vm.rs @@ -253,7 +253,9 @@ impl Vm { let action = Action::from_u32(action_raw); debug!(target: "spacegate_plugin_wasm", http_ctx_id, ?action, "on_request_headers returned"); - if action == Action::Pause { + // Guest 可能在 headers 阶段 Pause 以等待 on_request_body（尚未 dispatch_http_call）； + // 此时 pending_calls 为空，不能进入 drive_until_continue，否则会永久阻塞在 dispatch_rx。 + if action == Action::Pause && !self.store.data().pending_calls.is_empty() { self.drive_until_continue(http_ctx_id).await?; } @@ -446,6 +448,10 @@ impl Vm { if ctx.continue_requested && st.pending_calls.is_empty() { return Ok(()); } + // 无 outbound call 的 Pause（例如 defer 到 body hook）不应阻塞等待 dispatch 结果。 + if st.pending_calls.is_empty() { + return Ok(()); + } } let Some((token, result)) = self.dispatch_rx.recv().await else { return Err(WasmHostError::Dispatch("dispatch channel closed".to_string())); diff --git a/plugins/wasm/ai-gateway-queue/src/lib.rs b/plugins/wasm/ai-gateway-queue/src/lib.rs index 84757436..387e75e7 100644 --- a/plugins/wasm/ai-gateway-queue/src/lib.rs +++ b/plugins/wasm/ai-gateway-queue/src/lib.rs @@ -373,14 +373,14 @@ impl AiGatewayHttp { if status == 200 { let retry_after_ms = extract_json_number(&text, "retry_after_ms").unwrap_or(1000); let retry_after_secs = ((retry_after_ms + 999) / 1000).max(1).to_string(); - let retry_after_ms = retry_after_ms.to_string(); + let body = format!(r#"{{"error":"rate_limited","retry_after_ms":{retry_after_ms}}}"#); let headers = [ ("content-type".to_string(), "application/json".to_string()), ("retry-after".to_string(), retry_after_secs), - ("x-ratelimit-retry-after-ms".to_string(), retry_after_ms), + ("x-ratelimit-retry-after-ms".to_string(), retry_after_ms.to_string()), ]; let headers = headers.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect::>(); - self.send_http_response(429, headers, Some(text.as_bytes())); + self.send_http_response(429, headers, Some(body.as_bytes())); } else { self.send_json(502, r#"{"error":"rate_limit_service_error"}"#); } diff --git a/resource/ai-gateway-demo/config.json b/resource/ai-gateway-demo/config.json new file mode 100644 index 00000000..6195e0f0 --- /dev/null +++ b/resource/ai-gateway-demo/config.json @@ -0,0 +1,5 @@ +{ + "gateways": {}, + "plugins": {}, + "api_port": 19880 +} diff --git a/resource/ai-gateway-demo/gateway/ai-demo/config.json b/resource/ai-gateway-demo/gateway/ai-demo/config.json new file mode 100644 index 00000000..844ec103 --- /dev/null +++ b/resource/ai-gateway-demo/gateway/ai-demo/config.json @@ -0,0 +1,23 @@ +{ + "gateway": { + "name": "ai-demo", + "parameters": {}, + "listeners": [ + { + "name": "http", + "ip": "127.0.0.1", + "port": 9993, + "protocol": { + "type": "http" + } + } + ], + "plugins": [ + { + "code": "wasm", + "kind": "named", + "name": "ai-gateway-queue" + } + ] + } +} diff --git a/resource/ai-gateway-demo/gateway/ai-demo/route/ai.json b/resource/ai-gateway-demo/gateway/ai-demo/route/ai.json new file mode 100644 index 00000000..061381cc --- /dev/null +++ b/resource/ai-gateway-demo/gateway/ai-demo/route/ai.json @@ -0,0 +1,33 @@ +{ + "route_name": "ai", + "rules": [ + { + "matches": [ + { + "path": { + "kind": "Prefix", + "value": "/v1/" + } + } + ], + "plugins": [ + { + "code": "wasm", + "kind": "named", + "name": "ai-gateway-queue" + } + ], + "backends": [ + { + "host": { + "kind": "Host", + "host": "127.0.0.1" + }, + "port": 9000, + "weight": 1 + } + ] + } + ], + "priority": 0 +} diff --git a/resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json b/resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json index 4eb0b630..977ff1ec 100644 --- a/resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json +++ b/resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json @@ -1,5 +1,5 @@ { - "url": "plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm", + "url": "file:///Users/sh.zhang/Workspace/huayun/jiyan/ai-gateway-dev/spacegate/plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm", "validate_on_create": false, "fail_strategy": "fail_close", "plugin_name": "ai-gateway-queue", From 15c47611fc4ec597cbfd9924fe80fc1463ce44a6 Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Sun, 24 May 2026 21:25:13 +0800 Subject: [PATCH 15/19] feat: add AI gateway queue tests, K8s deploy stack and deployment docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 补齐集成测试与测试规格、K8s manifest/部署脚本及 README，并完善 ai-gateway-service 与 Wasm 插件配置能力，便于本地与集群环境验证队列限流。 Co-authored-by: Cursor --- .dockerignore | 5 +- binary/ai-gateway-service/Cargo.toml | 19 + binary/ai-gateway-service/README.md | 84 ++- .../config/ai-gateway-service.example.toml | 113 +++ .../config/ai-gateway-service.toml | 30 + .../scripts/queue-object-store-e2e.sh | 92 +++ .../scripts/run-gateway-e2e.sh | 7 + .../scripts/run-hurl-tests.sh | 96 +++ .../scripts/run-integration-tests.sh | 26 + .../scripts/run-wasm-policy-tests.sh | 7 + binary/ai-gateway-service/src/app.rs | 11 +- binary/ai-gateway-service/src/app/admin.rs | 9 +- binary/ai-gateway-service/src/app/callback.rs | 6 +- binary/ai-gateway-service/src/app/config.rs | 492 +++++++++++++ binary/ai-gateway-service/src/app/handlers.rs | 62 +- .../src/app/object_store.rs | 121 ++-- binary/ai-gateway-service/src/app/queue.rs | 104 +-- .../ai-gateway-service/src/app/ratelimit.rs | 53 +- binary/ai-gateway-service/src/app/runtime.rs | 42 +- .../src/app/test_support.rs | 366 ++++++++++ binary/ai-gateway-service/src/app/tests.rs | 18 +- binary/ai-gateway-service/src/app/types.rs | 264 ++++++- binary/ai-gateway-service/src/app/util.rs | 19 +- .../src/app/wait_subscriber.rs | 59 ++ binary/ai-gateway-service/src/lib.rs | 5 + binary/ai-gateway-service/src/main.rs | 4 +- .../tests/fixtures/small.json | 1 + .../ai-gateway-service/tests/hurl/admin.hurl | 31 + .../tests/hurl/metrics.hurl | 13 + .../ai-gateway-service/tests/hurl/queue.hurl | 21 + .../tests/hurl/ratelimit.hurl | 15 + .../ai-gateway-service/tests/hurl/wait.hurl | 15 + .../tests/integration/admin_tenant_limit.rs | 64 ++ .../tests/integration/body_store.rs | 39 ++ .../tests/integration/common.rs | 9 + .../tests/integration/enqueue_queue.rs | 108 +++ .../tests/integration/enqueue_wait.rs | 54 ++ .../tests/integration/metrics.rs | 35 + .../tests/integration/mod.rs | 9 + .../tests/integration/ratelimit.rs | 70 ++ .../tests/integration/worker_reliability.rs | 42 ++ deploy/README.md | 502 ++++++++++++++ deploy/k8s/ai-gateway/ai-gateway-service.yaml | 71 ++ deploy/k8s/ai-gateway/apply.sh | 43 ++ deploy/k8s/ai-gateway/build-images.sh | 18 + .../docker/Dockerfile.ai-gateway-service | 17 + deploy/k8s/ai-gateway/files/.gitkeep | 2 + deploy/k8s/ai-gateway/gateway-ai.yaml | 17 + deploy/k8s/ai-gateway/httproute-ai.yaml | 20 + deploy/k8s/ai-gateway/kustomization.yaml | 25 + deploy/k8s/ai-gateway/mock-upstream.yaml | 49 ++ deploy/k8s/ai-gateway/redis.yaml | 54 ++ .../ai-gateway/sgfilter-ai-gateway-queue.yaml | 42 ++ deploy/k8s/ai-gateway/verify.sh | 50 ++ deploy/k8s/ai-gateway/wasm-server.yaml | 57 ++ .../wasmplugin-ai-gateway-queue.yaml | 27 + deploy/push-wasm-oci.sh | 35 + docs/ai-gateway-queue-design-gap-fixlist.md | 644 ++++++++++++++++++ docs/ai-gateway-queue-test-spec.md | 370 ++++++++++ plugins/wasm/ai-gateway-queue/Cargo.toml | 2 +- plugins/wasm/ai-gateway-queue/README.md | 25 +- plugins/wasm/ai-gateway-queue/src/lib.rs | 160 ++--- plugins/wasm/ai-gateway-queue/src/policy.rs | 68 ++ .../plugin/wasm.ai-gateway-queue.local.json | 29 + 64 files changed, 4674 insertions(+), 293 deletions(-) create mode 100644 binary/ai-gateway-service/config/ai-gateway-service.example.toml create mode 100644 binary/ai-gateway-service/config/ai-gateway-service.toml create mode 100755 binary/ai-gateway-service/scripts/queue-object-store-e2e.sh create mode 100755 binary/ai-gateway-service/scripts/run-gateway-e2e.sh create mode 100755 binary/ai-gateway-service/scripts/run-hurl-tests.sh create mode 100755 binary/ai-gateway-service/scripts/run-integration-tests.sh create mode 100755 binary/ai-gateway-service/scripts/run-wasm-policy-tests.sh create mode 100644 binary/ai-gateway-service/src/app/config.rs create mode 100644 binary/ai-gateway-service/src/app/test_support.rs create mode 100644 binary/ai-gateway-service/src/app/wait_subscriber.rs create mode 100644 binary/ai-gateway-service/src/lib.rs create mode 100644 binary/ai-gateway-service/tests/fixtures/small.json create mode 100644 binary/ai-gateway-service/tests/hurl/admin.hurl create mode 100644 binary/ai-gateway-service/tests/hurl/metrics.hurl create mode 100644 binary/ai-gateway-service/tests/hurl/queue.hurl create mode 100644 binary/ai-gateway-service/tests/hurl/ratelimit.hurl create mode 100644 binary/ai-gateway-service/tests/hurl/wait.hurl create mode 100644 binary/ai-gateway-service/tests/integration/admin_tenant_limit.rs create mode 100644 binary/ai-gateway-service/tests/integration/body_store.rs create mode 100644 binary/ai-gateway-service/tests/integration/common.rs create mode 100644 binary/ai-gateway-service/tests/integration/enqueue_queue.rs create mode 100644 binary/ai-gateway-service/tests/integration/enqueue_wait.rs create mode 100644 binary/ai-gateway-service/tests/integration/metrics.rs create mode 100644 binary/ai-gateway-service/tests/integration/mod.rs create mode 100644 binary/ai-gateway-service/tests/integration/ratelimit.rs create mode 100644 binary/ai-gateway-service/tests/integration/worker_reliability.rs create mode 100644 deploy/README.md create mode 100644 deploy/k8s/ai-gateway/ai-gateway-service.yaml create mode 100755 deploy/k8s/ai-gateway/apply.sh create mode 100755 deploy/k8s/ai-gateway/build-images.sh create mode 100644 deploy/k8s/ai-gateway/docker/Dockerfile.ai-gateway-service create mode 100644 deploy/k8s/ai-gateway/files/.gitkeep create mode 100644 deploy/k8s/ai-gateway/gateway-ai.yaml create mode 100644 deploy/k8s/ai-gateway/httproute-ai.yaml create mode 100644 deploy/k8s/ai-gateway/kustomization.yaml create mode 100644 deploy/k8s/ai-gateway/mock-upstream.yaml create mode 100644 deploy/k8s/ai-gateway/redis.yaml create mode 100644 deploy/k8s/ai-gateway/sgfilter-ai-gateway-queue.yaml create mode 100755 deploy/k8s/ai-gateway/verify.sh create mode 100644 deploy/k8s/ai-gateway/wasm-server.yaml create mode 100644 deploy/k8s/ai-gateway/wasmplugin-ai-gateway-queue.yaml create mode 100755 deploy/push-wasm-oci.sh create mode 100644 docs/ai-gateway-queue-design-gap-fixlist.md create mode 100644 docs/ai-gateway-queue-test-spec.md create mode 100644 plugins/wasm/ai-gateway-queue/src/policy.rs create mode 100644 resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.local.json diff --git a/.dockerignore b/.dockerignore index 568383a0..658f9413 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,6 @@ target +**/target .git -**/*.md +**/.git +.DS_Store +**/.DS_Store diff --git a/binary/ai-gateway-service/Cargo.toml b/binary/ai-gateway-service/Cargo.toml index 49efd520..6e3e0f97 100644 --- a/binary/ai-gateway-service/Cargo.toml +++ b/binary/ai-gateway-service/Cargo.toml @@ -24,7 +24,26 @@ reqwest = { workspace = true, features = ["json"] } schemars = "0.8" serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +toml = { workspace = true } tokio = { workspace = true, features = ["full"] } tower-http = { version = "0.6", features = ["cors", "trace"] } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } +ulid = "1.1" + +[features] +test-support = [] + +[lib] +name = "ai_gateway_service" +path = "src/lib.rs" + +[[test]] +name = "integration" +path = "tests/integration/mod.rs" +required-features = ["test-support"] + +[dev-dependencies] +tempfile = "3" +serde_json = { workspace = true } +reqwest = { workspace = true, features = ["json"] } diff --git a/binary/ai-gateway-service/README.md b/binary/ai-gateway-service/README.md index b1da5609..16c7b664 100644 --- a/binary/ai-gateway-service/README.md +++ b/binary/ai-gateway-service/README.md @@ -7,10 +7,10 @@ It keeps Redis, worker execution, Pub/Sub waiting, callback delivery, and result ## Endpoints - `POST /v1/ratelimit/check` - - Reads `X-Tenant-Id`, `X-Model`, and `X-Original-Path`. - - Runs a Redis Lua token bucket. - - Can override per tenant with Redis keys `ai:tenant:ratelimit:{tenant}:rps` and `ai:tenant:ratelimit:{tenant}:burst`. - - Returns `{ "allowed": bool, "retry_after_ms": number }`. + - Reads `X-Tenant-Id`, optional `X-Model`, `X-Original-Path`, and `X-RateLimit-Policy`. + - Runs a Redis Lua token bucket keyed by **tenant only** (`ai:ratelimit:{tenant}:tokens/ts`). + - Per-tenant overrides via Admin API or Redis keys under `ai:tenant:ratelimit:{tenant}[:model:...][:path:...][:policy:...]`. + - Returns `{ "allowed": bool, "retry_after_ms": number }`. Wasm calls this for **all** policies before enqueue or upstream passthrough. - `POST /v1/queue/enqueue` - Requires `X-Callback-URL` by default. - Streams the request body, then stores either inline base64 body or an object-store reference in Redis Stream. @@ -18,8 +18,9 @@ It keeps Redis, worker execution, Pub/Sub waiting, callback delivery, and result - `POST /v1/queue/enqueue-and-wait` - Enqueues the job and waits for the worker result via Redis Pub/Sub. - Returns the upstream response with `X-Job-Id` and `X-Queue-Wait-Ms`, or `504`. -- `GET /v1/jobs/{job_id}` - - Returns the stored result JSON while the result key TTL is alive. +- `GET /v1/jobs/{job_id}` / `GET /jobs/{job_id}/status` + - When the job is completed, returns the **raw upstream HTTP response** (status, headers, body) with `X-Job-Id`. + - While pending or on error, returns JSON status metadata. - `GET /metrics` - Returns Prometheus text metrics for queue depth, PEL size, DLQ depth, enqueue latency, body size, waits, limits, callbacks, retries, object offload, and worker counters. @@ -31,6 +32,39 @@ cargo run -p ai-gateway-service -- \ --upstream-base-url http://127.0.0.1:9000 ``` +Or use a TOML config file (recommended for local / deployment): + +```bash +cargo run -p ai-gateway-service -- --config config/ai-gateway-service.toml +``` + +If `--config` / `AI_GATEWAY_CONFIG` is omitted, the service looks for `ai-gateway-service.toml` in the **same directory as the executable**. For deployment, place the binary and config file together: + +```text +/opt/ai-gateway/ + ai-gateway-service # binary + ai-gateway-service.toml # auto-loaded +``` + +Example configs live under `config/`: + +- `config/ai-gateway-service.example.toml` — full reference with all sections +- `config/ai-gateway-service.toml` — minimal local dev template + +Precedence: explicit CLI flags / environment variables > config file > built-in defaults. + +Default config discovery order: + +1. `--config` or `AI_GATEWAY_CONFIG` +2. `{executable_dir}/ai-gateway-service.toml` (if the file exists) +3. Built-in defaults only + +Set the config path via environment variable: + +```bash +AI_GATEWAY_CONFIG=config/ai-gateway-service.toml cargo run -p ai-gateway-service +``` + Useful environment variables: ```bash @@ -82,7 +116,13 @@ CreateMultipartUpload -> UploadPart* -> CompleteMultipartUpload If any part upload or completion fails, the service sends `AbortMultipartUpload` before returning the enqueue error. The current implementation expects a MinIO/S3-compatible endpoint that accepts either unsigned requests or the configured static auth header. -Tenant rate-limit config can be overridden without restarting the service. The service checks Redis keys from most-specific to least-specific, using JSON or CSV values: +Tenant rate-limit overrides (Admin API + Redis): + +```text +GET/PUT/DELETE /v1/admin/tenant-rate-limits +``` + +Redis key patterns (most specific match wins; token bucket remains tenant-scoped): ```text ai:tenant:ratelimit:{tenant}:model:{model}:path:{path}:policy:{policy} @@ -109,7 +149,17 @@ CSV value: The old per-tenant keys are still supported as fallback: `ai:tenant:ratelimit:{tenant}:rps`, `:burst`, and `:cost`. -Priority queues are disabled by default. Enable them and send `X-Queue-Priority: high|normal|low` to route jobs to separate streams, or configure model/tenant defaults: +Global defaults when no tenant rule matches: + +```bash +AI_RATE_LIMIT_RPS=100 +AI_RATE_LIMIT_BURST=200 +AI_RATE_LIMIT_COST=1 +``` + +The Wasm plugin invokes `/v1/ratelimit/check` for **abandon**, **queue**, and **wait** before passthrough or enqueue. + +Priority streams are **enabled by default** (`AI_ENABLE_PRIORITY_STREAMS=true`). Send `X-Queue-Priority: high|normal|low` to route jobs to separate streams, or configure model/tenant defaults: ```bash AI_ENABLE_PRIORITY_STREAMS=true @@ -142,6 +192,24 @@ Unit tests (mock S3 multipart server, no Docker): cargo test -p ai-gateway-service store_body_ ``` +## 测试规格与集成测试 + +完整用例规格见 [`spacegate/docs/ai-gateway-queue-test-spec.md`](../../docs/ai-gateway-queue-test-spec.md)（TC-* 编号，映射设计文档章节）。 + +```bash +# 单元测试（无需 Redis） +cd spacegate && cargo test -p ai-gateway-service + +# Rust 集成测试（需 Redis 7+） +./spacegate/binary/ai-gateway-service/scripts/run-integration-tests.sh + +# Hurl 黑盒（需 hurl + Redis + 编译 release binary） +./spacegate/binary/ai-gateway-service/scripts/run-hurl-tests.sh + +# Wasm 策略纯逻辑（host 侧） +./spacegate/binary/ai-gateway-service/scripts/run-wasm-policy-tests.sh +``` + MinIO end-to-end (Docker + worker roundtrip): ```bash diff --git a/binary/ai-gateway-service/config/ai-gateway-service.example.toml b/binary/ai-gateway-service/config/ai-gateway-service.example.toml new file mode 100644 index 00000000..bdcc59cc --- /dev/null +++ b/binary/ai-gateway-service/config/ai-gateway-service.example.toml @@ -0,0 +1,113 @@ +# ai-gateway-service 配置文件示例 +# 用法： +# cargo run -p ai-gateway-service -- --config config/ai-gateway-service.toml +# AI_GATEWAY_CONFIG=config/ai-gateway-service.toml cargo run -p ai-gateway-service +# +# 优先级：显式 CLI 参数 / 环境变量 > 本配置文件 > 内置默认值 + +# --------------------------------------------------------------------------- +# 服务监听 +# --------------------------------------------------------------------------- +[server] +host = "0.0.0.0" +port = 18080 + +# --------------------------------------------------------------------------- +# Redis（队列、限流、结果存储的核心依赖） +# --------------------------------------------------------------------------- +[redis] +# 单机示例 +url = "redis://127.0.0.1/" +# 带密码 / 指定 DB 示例： +# url = "redis://:your-password@redis.example.com:6379/0" + +# --------------------------------------------------------------------------- +# 上游 AI 服务（Worker 消费队列后转发到此地址） +# --------------------------------------------------------------------------- +[upstream] +base_url = "http://127.0.0.1:9000" + +# --------------------------------------------------------------------------- +# 队列 Stream 与优先级 +# --------------------------------------------------------------------------- +[queue] +stream = "ai:jobs" +high_stream = "ai:jobs:high" +low_stream = "ai:jobs:low" +enable_priority_streams = true +default_priority = "normal" +high_models = ["gpt-4o", "qwen-max"] +low_tenants = ["free"] +high_weight = 3 +normal_weight = 1 +low_weight = 1 +max_len = 100000 +group = "ai-gateway-workers" +consumer = "ai-gateway-service" +job_dlq_stream = "ai:job-dlq" + +# --------------------------------------------------------------------------- +# V1 全局限流（仅 abandon 路径生效；租户差异化配额为 V2，见 README） +# --------------------------------------------------------------------------- +[rate_limit] +rps = 100 +burst = 200 +cost = 1 +tenant_prefix = "ai:tenant:ratelimit:" + +# --------------------------------------------------------------------------- +# Worker 与任务回收 +# --------------------------------------------------------------------------- +[worker] +concurrency = 10 +wait_timeout_secs = 60 +reclaim_interval_secs = 30 +reclaim_min_idle_secs = 30 +job_process_lease_secs = 120 +job_max_delivery_attempts = 5 + +# --------------------------------------------------------------------------- +# 回调（queue 模式） +# --------------------------------------------------------------------------- +[callback] +require_https = true +max_retry_attempts = 5 +retry_initial_delay_ms = 1000 +retry_max_delay_ms = 60000 +retry_reclaim_idle_secs = 60 +retry_stream = "ai:callback-retry" +retry_group = "ai-gateway-callbacks" +dlq_stream = "ai:callback-dlq" + +# --------------------------------------------------------------------------- +# 结果缓存（wait 模式 / 轮询） +# --------------------------------------------------------------------------- +[result] +key_prefix = "result:" +channel_prefix = "result:" +ttl_secs = 120 + +# --------------------------------------------------------------------------- +# 请求体大小限制 +# --------------------------------------------------------------------------- +[body] +max_bytes = 33554432 # 32 MiB +inline_threshold = 131072 # 128 KiB 以下 inline 存 Redis +read_concurrency = 200 + +# --------------------------------------------------------------------------- +# 大 Body 对象存储（可选，S3 / MinIO 兼容） +# --------------------------------------------------------------------------- +[object_store] +# endpoint = "http://127.0.0.1:9000" +bucket = "ai-gateway-body" +prefix = "bodies" +multipart_part_size = 5242880 +# auth_header = "Authorization: Bearer your-token" + +# --------------------------------------------------------------------------- +# Admin API CORS（本地开发可留空，表示 permissive） +# --------------------------------------------------------------------------- +[admin] +cors_origins = [] +# cors_origins = ["http://localhost:5173", "http://127.0.0.1:5173"] diff --git a/binary/ai-gateway-service/config/ai-gateway-service.toml b/binary/ai-gateway-service/config/ai-gateway-service.toml new file mode 100644 index 00000000..30f691e0 --- /dev/null +++ b/binary/ai-gateway-service/config/ai-gateway-service.toml @@ -0,0 +1,30 @@ +# 本地开发配置：按需修改 Redis / 上游 / 对象存储地址 + +[server] +host = "0.0.0.0" +port = 18080 + +[redis] +url = "redis://127.0.0.1/" + +[upstream] +base_url = "http://127.0.0.1:9000" + +[callback] +require_https = false + +# 请求体大小：超过 inline_threshold 且配置了 object_store.endpoint 时走 MinIO/S3 +[body] +max_bytes = 33554432 # 32 MiB，与 Wasm 插件 limits.max_body_bytes 保持一致 +inline_threshold = 131072 # 128 KiB 以下 inline 存 Redis +read_concurrency = 200 + +# 大 Body 对象存储（S3 / MinIO 兼容） +# 本地 MinIO 可参考 tests/queue-object-store-e2e.sh，默认端口 9001（避免与 mock 上游 :9000 冲突） +[object_store] +endpoint = "http://127.0.0.1:9001" +bucket = "ai-gateway-body" +prefix = "bodies" +multipart_part_size = 5242880 +# 无鉴权 MinIO 可留空；需要鉴权时使用 "Header-Name: value" 格式 +# auth_header = "Authorization: Bearer your-token" diff --git a/binary/ai-gateway-service/scripts/queue-object-store-e2e.sh b/binary/ai-gateway-service/scripts/queue-object-store-e2e.sh new file mode 100755 index 00000000..879510b9 --- /dev/null +++ b/binary/ai-gateway-service/scripts/queue-object-store-e2e.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# MinIO + 大 body E2E（TC-BODY-02 smoke） +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +WORKSPACE="$(cd "$ROOT/../.." && pwd)" +cd "$WORKSPACE" + +REDIS_URL="${REDIS_URL:-redis://127.0.0.1/}" +MINIO_PORT="${MINIO_PORT:-9001}" +SVC_PORT="${E2E_SERVICE_PORT:-18081}" + +if ! command -v docker >/dev/null 2>&1; then + echo "SKIP: docker not available" >&2 + exit 0 +fi + +docker rm -f ai-gateway-minio-e2e 2>/dev/null || true +docker run -d --name ai-gateway-minio-e2e \ + -p "${MINIO_PORT}:9000" \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data >/dev/null + +cleanup() { + docker rm -f ai-gateway-minio-e2e 2>/dev/null || true + kill $SVC_PID $UP_PID 2>/dev/null || true +} +trap cleanup EXIT + +sleep 2 + +# MinIO 需先创建 bucket 并设为 public（服务使用无 SigV4 的直传 HTTP） +if docker run --rm --network host --entrypoint /bin/sh minio/mc -c \ + "mc alias set local http://127.0.0.1:${MINIO_PORT} minioadmin minioadmin && \ + mc mb --ignore-existing local/ai-gateway-body && \ + mc anonymous set public local/ai-gateway-body" >/dev/null 2>&1; then + echo "MinIO bucket ai-gateway-body ready (public)." +else + echo "ERROR: MinIO bucket bootstrap failed" >&2 + exit 1 +fi + +python3 - </dev/null 2>&1; then + echo "ERROR: hurl not installed. See https://hurl.dev" >&2 + exit 1 +fi + +redis_ok=false +if command -v redis-cli >/dev/null 2>&1 && redis-cli -u "$REDIS_URL" PING >/dev/null 2>&1; then + redis_ok=true +elif docker exec ai-gateway-redis redis-cli PING >/dev/null 2>&1; then + redis_ok=true +elif nc -z 127.0.0.1 6379 2>/dev/null; then + redis_ok=true +fi +if [[ "$redis_ok" != true ]]; then + echo "ERROR: Redis not reachable at $REDIS_URL" >&2 + exit 1 +fi + +# mock upstream :9000 +python3 - <<'PY' & +import json +from http.server import BaseHTTPRequestHandler, HTTPServer + +class H(BaseHTTPRequestHandler): + def do_POST(self): + body = json.dumps({"upstream": True, "hurl": True}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): pass + +HTTPServer(("127.0.0.1", 9000), H).serve_forever() +PY +UP_PID=$! + +# mock callback +python3 - <<'PY' & +from http.server import BaseHTTPRequestHandler, HTTPServer + +class H(BaseHTTPRequestHandler): + def do_POST(self): + n = int(self.headers.get("Content-Length", 0)) + self.rfile.read(n) + self.send_response(200) + self.end_headers() + def log_message(self, *args): pass + +HTTPServer(("127.0.0.1", 9002), H).serve_forever() +PY +CB_PID=$! + +cleanup() { + kill $UP_PID $CB_PID $SVC_PID 2>/dev/null || true +} +trap cleanup EXIT + +cargo build -q -p ai-gateway-service --release +SVC="$WORKSPACE/target/release/ai-gateway-service" +PORT="${HURL_SERVICE_PORT:-18090}" +CALLBACK="http://127.0.0.1:9002/cb" + +"$SVC" \ + --redis-url "$REDIS_URL" \ + --port "$PORT" \ + --host 127.0.0.1 \ + --upstream-base-url http://127.0.0.1:9000 \ + & +SVC_PID=$! +sleep 1 + +export service_url="http://127.0.0.1:${PORT}" +export callback_url="$CALLBACK" + +hurl --test \ + --variable service_url="$service_url" \ + --variable callback_url="$callback_url" \ + --file-root "$ROOT/tests/fixtures" \ + "$ROOT/tests/hurl/ratelimit.hurl" \ + "$ROOT/tests/hurl/queue.hurl" \ + "$ROOT/tests/hurl/wait.hurl" \ + "$ROOT/tests/hurl/metrics.hurl" \ + "$ROOT/tests/hurl/admin.hurl" + +echo "Hurl tests passed." diff --git a/binary/ai-gateway-service/scripts/run-integration-tests.sh b/binary/ai-gateway-service/scripts/run-integration-tests.sh new file mode 100755 index 00000000..5b69c10b --- /dev/null +++ b/binary/ai-gateway-service/scripts/run-integration-tests.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# 运行 Rust 集成测试（需 Redis 7+） +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT/../.." + +REDIS_URL="${REDIS_URL:-redis://127.0.0.1/}" +export REDIS_URL + +echo "Checking Redis at $REDIS_URL ..." +redis_ok=false +if command -v redis-cli >/dev/null 2>&1; then + if redis-cli -u "$REDIS_URL" INFO server 2>/dev/null | grep -qE 'redis_version:(7|[89])'; then + redis_ok=true + fi +elif docker exec ai-gateway-redis redis-cli INFO server 2>/dev/null | grep -qE 'redis_version:(7|[89])'; then + redis_ok=true +elif nc -z 127.0.0.1 6379 2>/dev/null; then + redis_ok=true +fi +if [[ "$redis_ok" != true ]]; then + echo "ERROR: Redis 7+ required. Start redis or set REDIS_URL." >&2 + exit 1 +fi + +cargo test -p ai-gateway-service --features test-support --test integration "$@" diff --git a/binary/ai-gateway-service/scripts/run-wasm-policy-tests.sh b/binary/ai-gateway-service/scripts/run-wasm-policy-tests.sh new file mode 100755 index 00000000..04d47754 --- /dev/null +++ b/binary/ai-gateway-service/scripts/run-wasm-policy-tests.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Wasm 策略纯逻辑 host 侧单测（绕过 wasm32 默认 target） +set -euo pipefail +DIR="$(cd "$(dirname "$0")/../../../plugins/wasm/ai-gateway-queue" && pwd)" +HOST=$(rustc -vV | sed -n 's/host: //p') +cd "$DIR" +cargo test --lib --target "$HOST" "$@" diff --git a/binary/ai-gateway-service/src/app.rs b/binary/ai-gateway-service/src/app.rs index 2b934515..5b424929 100644 --- a/binary/ai-gateway-service/src/app.rs +++ b/binary/ai-gateway-service/src/app.rs @@ -14,17 +14,19 @@ use base64::Engine; use clap::Parser; use fred::clients::{Client as FredClient, SubscriberClient}; use fred::prelude::*; +use fred::types::InfoKind; use fred::types::streams::XReadResponse; use fred::types::ExpireOptions; use futures_util::StreamExt; use schemars::{schema_for, JsonSchema}; use serde::{Deserialize, Serialize}; use std::sync::Mutex; -use tokio::sync::Semaphore; +use tokio::sync::{oneshot, Semaphore}; use tower_http::cors::{Any, CorsLayer}; use tower_http::trace::TraceLayer; include!("app/types.rs"); +include!("app/config.rs"); include!("app/runtime.rs"); include!("app/handlers.rs"); include!("app/queue.rs"); @@ -34,5 +36,12 @@ include!("app/object_store.rs"); include!("app/metrics.rs"); include!("app/ratelimit.rs"); include!("app/admin.rs"); +include!("app/wait_subscriber.rs"); include!("app/util.rs"); + +#[cfg(feature = "test-support")] +pub mod test_support { + include!("app/test_support.rs"); +} + include!("app/tests.rs"); diff --git a/binary/ai-gateway-service/src/app/admin.rs b/binary/ai-gateway-service/src/app/admin.rs index e60892e4..1ae18460 100644 --- a/binary/ai-gateway-service/src/app/admin.rs +++ b/binary/ai-gateway-service/src/app/admin.rs @@ -19,19 +19,18 @@ async fn admin_plugin_readme(Path(plugin): Path) -> Result, Query(filters): Query>) -> Result>, ServiceError> { - let rules = list_tenant_rate_limit_rules(&state, &filters).await?; - Ok(Json(rules)) + Ok(Json(list_tenant_rate_limit_rules(&state, &filters).await?)) } async fn admin_upsert_tenant_rate_limit(State(state): State, Json(rule): Json) -> Result, ServiceError> { - let rule = upsert_tenant_rate_limit_rule(&state, rule).await?; - Ok(Json(rule)) + Ok(Json(upsert_tenant_rate_limit_rule(&state, rule).await?)) } async fn admin_delete_tenant_rate_limit(State(state): State, Json(rule): Json) -> Result, ServiceError> { let removed = delete_tenant_rate_limit_rule(&state, rule).await?; - Ok(Json(serde_json::json!({ "deleted": removed }))) + Ok(Json(serde_json::json!({ "removed": removed }))) } fn add_ai_gateway_queue_schema_extensions(value: &mut serde_json::Value) { diff --git a/binary/ai-gateway-service/src/app/callback.rs b/binary/ai-gateway-service/src/app/callback.rs index dd3cbcc9..0836ad19 100644 --- a/binary/ai-gateway-service/src/app/callback.rs +++ b/binary/ai-gateway-service/src/app/callback.rs @@ -1,14 +1,10 @@ fn callback_body(result: &StoredResult) -> serde_json::Value { + // 设计文档回调 JSON：job_id / status / result / completed_at serde_json::json!({ "job_id": result.job_id, "status": result.status, - "http_status": result.http_status, - "headers": result.headers, "result": decode_callback_result(&result.body_base64), - "body_base64": result.body_base64, "completed_at": format_completed_at_rfc3339(result.completed_at_ms), - "completed_at_ms": result.completed_at_ms, - "error": result.error, }) } diff --git a/binary/ai-gateway-service/src/app/config.rs b/binary/ai-gateway-service/src/app/config.rs new file mode 100644 index 00000000..b602ee8e --- /dev/null +++ b/binary/ai-gateway-service/src/app/config.rs @@ -0,0 +1,492 @@ +use std::path::{Path as ConfigPath, PathBuf}; + +use clap::{parser::ValueSource, ArgMatches, CommandFactory, FromArgMatches}; + +/// 默认可执行文件同目录下的配置文件名。 +const DEFAULT_CONFIG_FILE_NAME: &str = "ai-gateway-service.toml"; + +/// CLI 包装层：配置文件路径 + 原有 Args。 +#[derive(Debug, Parser)] +#[command(version, about = "External Redis-backed rate-limit and queue service for SpaceGate AI gateway")] +struct Cli { + /// TOML 配置文件路径；未指定时尝试读取可执行文件同目录下的 ai-gateway-service.toml。 + #[arg(long, env = "AI_GATEWAY_CONFIG", value_name = "FILE")] + config: Option, + #[command(flatten)] + args: Args, +} + +/// 解析最终使用的配置文件路径：显式参数 > 可执行文件同目录默认文件。 +fn resolve_config_path(explicit: Option) -> Option { + if let Some(path) = explicit { + return Some(path); + } + default_config_path_beside_executable() +} + +/// 可执行文件所在目录下的默认配置文件（存在才返回）。 +fn default_config_path_beside_executable() -> Option { + std::env::current_exe().ok().and_then(|exe| default_config_path_in_dir(&exe)) +} + +/// 给定可执行文件路径，返回同目录下默认配置文件路径（存在才返回）。 +fn default_config_path_in_dir(exe_path: &ConfigPath) -> Option { + let dir = exe_path.parent()?; + let path = dir.join(DEFAULT_CONFIG_FILE_NAME); + path.is_file().then_some(path) +} + +/// 从 CLI、环境变量和可选 TOML 配置文件合并出最终运行参数。 +fn load_args() -> Result> { + let matches = Cli::command().get_matches(); + let explicit_config = matches.get_one::("config").cloned(); + let config_path = resolve_config_path(explicit_config); + let cli = Cli::from_arg_matches(&matches).expect("cli args"); + + let file_args = match config_path.as_deref() { + Some(path) => { + tracing::info!(path = %path.display(), "loading config file"); + Some(ServiceConfigFile::load(path)?.into_args()) + } + None => None, + }; + + Ok(merge_args(file_args, cli.args, &matches)) +} + +/// TOML 配置文件根结构；各 section 均可选，便于按需扩展。 +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct ServiceConfigFile { + server: ServerSection, + redis: RedisSection, + upstream: UpstreamSection, + queue: QueueSection, + rate_limit: RateLimitSection, + worker: WorkerSection, + callback: CallbackSection, + result: ResultSection, + body: BodySection, + object_store: ObjectStoreSection, + admin: AdminSection, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct ServerSection { + host: Option, + port: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct RedisSection { + /// Redis 连接 URL，例如 redis://127.0.0.1/ 或 redis://:password@host:6379/0 + url: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct UpstreamSection { + /// 上游 AI 服务地址；未配置时只入队，不启动 worker。 + base_url: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct QueueSection { + stream: Option, + high_stream: Option, + low_stream: Option, + enable_priority_streams: Option, + default_priority: Option, + high_models: Option>, + low_models: Option>, + high_tenants: Option>, + low_tenants: Option>, + high_weight: Option, + normal_weight: Option, + low_weight: Option, + max_len: Option, + group: Option, + consumer: Option, + job_dlq_stream: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct RateLimitSection { + rps: Option, + burst: Option, + cost: Option, + tenant_prefix: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct WorkerSection { + concurrency: Option, + wait_timeout_secs: Option, + reclaim_interval_secs: Option, + reclaim_min_idle_secs: Option, + job_process_lease_secs: Option, + job_max_delivery_attempts: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct CallbackSection { + require_https: Option, + max_retry_attempts: Option, + retry_initial_delay_ms: Option, + retry_max_delay_ms: Option, + retry_reclaim_idle_secs: Option, + retry_stream: Option, + retry_group: Option, + dlq_stream: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct ResultSection { + key_prefix: Option, + channel_prefix: Option, + ttl_secs: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct BodySection { + max_bytes: Option, + inline_threshold: Option, + read_concurrency: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct ObjectStoreSection { + endpoint: Option, + bucket: Option, + prefix: Option, + multipart_part_size: Option, + auth_header: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct AdminSection { + cors_origins: Option>, +} + +impl ServiceConfigFile { + fn load(path: &ConfigPath) -> Result> { + let raw = std::fs::read_to_string(path).map_err(|e| format!("read config `{}`: {e}", path.display()))?; + let cfg: Self = toml::from_str(&raw).map_err(|e| format!("parse config `{}`: {e}", path.display()))?; + Ok(cfg) + } + + fn into_args(self) -> Args { + let mut args = Args::default(); + if let Some(host) = self.server.host { + args.host = host.parse().unwrap_or(args.host); + } + if let Some(port) = self.server.port { + args.port = port; + } + if let Some(url) = self.redis.url { + args.redis_url = url; + } + args.upstream_base_url = self.upstream.base_url; + + if let Some(stream) = self.queue.stream { + args.stream_key = stream; + } + if let Some(stream) = self.queue.high_stream { + args.high_priority_stream_key = stream; + } + if let Some(stream) = self.queue.low_stream { + args.low_priority_stream_key = stream; + } + if let Some(value) = self.queue.enable_priority_streams { + args.enable_priority_streams = value; + } + if let Some(value) = self.queue.default_priority { + args.queue_default_priority = value; + } + if let Some(values) = self.queue.high_models { + args.queue_high_models = join_csv(&values); + } + if let Some(values) = self.queue.low_models { + args.queue_low_models = join_csv(&values); + } + if let Some(values) = self.queue.high_tenants { + args.queue_high_tenants = join_csv(&values); + } + if let Some(values) = self.queue.low_tenants { + args.queue_low_tenants = join_csv(&values); + } + if let Some(value) = self.queue.high_weight { + args.queue_high_weight = value; + } + if let Some(value) = self.queue.normal_weight { + args.queue_normal_weight = value; + } + if let Some(value) = self.queue.low_weight { + args.queue_low_weight = value; + } + if let Some(value) = self.queue.max_len { + args.stream_max_len = value; + } + if let Some(value) = self.queue.group { + args.consumer_group = value; + } + if let Some(value) = self.queue.consumer { + args.consumer_name = value; + } + if let Some(value) = self.queue.job_dlq_stream { + args.job_dlq_stream = value; + } + + if let Some(value) = self.rate_limit.rps { + args.rate_limit_rps = value; + } + if let Some(value) = self.rate_limit.burst { + args.rate_limit_burst = value; + } + if let Some(value) = self.rate_limit.cost { + args.rate_limit_cost = value; + } + if let Some(value) = self.rate_limit.tenant_prefix { + args.tenant_rate_limit_prefix = value; + } + + if let Some(value) = self.worker.concurrency { + args.worker_concurrency = value; + } + if let Some(value) = self.worker.wait_timeout_secs { + args.wait_timeout_secs = value; + } + if let Some(value) = self.worker.reclaim_interval_secs { + args.reclaim_interval_secs = value; + } + if let Some(value) = self.worker.reclaim_min_idle_secs { + args.reclaim_min_idle_secs = value; + } + if let Some(value) = self.worker.job_process_lease_secs { + args.job_process_lease_secs = value; + } + if let Some(value) = self.worker.job_max_delivery_attempts { + args.job_max_delivery_attempts = value; + } + + if let Some(value) = self.callback.require_https { + args.require_https_callback = value; + } + if let Some(value) = self.callback.max_retry_attempts { + args.callback_max_retry_attempts = value; + } + if let Some(value) = self.callback.retry_initial_delay_ms { + args.callback_retry_initial_delay_ms = value; + } + if let Some(value) = self.callback.retry_max_delay_ms { + args.callback_retry_max_delay_ms = value; + } + if let Some(value) = self.callback.retry_reclaim_idle_secs { + args.callback_retry_reclaim_idle_secs = value; + } + if let Some(value) = self.callback.retry_stream { + args.callback_retry_stream = value; + } + if let Some(value) = self.callback.retry_group { + args.callback_retry_group = value; + } + if let Some(value) = self.callback.dlq_stream { + args.callback_dlq_stream = value; + } + + if let Some(value) = self.result.key_prefix { + args.result_key_prefix = value; + } + if let Some(value) = self.result.channel_prefix { + args.result_channel_prefix = value; + } + if let Some(value) = self.result.ttl_secs { + args.result_ttl_secs = value; + } + + if let Some(value) = self.body.max_bytes { + args.max_body_bytes = value; + } + if let Some(value) = self.body.inline_threshold { + args.inline_threshold = value; + } + if let Some(value) = self.body.read_concurrency { + args.body_read_concurrency = value; + } + + args.object_store_endpoint = self.object_store.endpoint; + if let Some(value) = self.object_store.bucket { + args.object_store_bucket = value; + } + if let Some(value) = self.object_store.prefix { + args.object_store_prefix = value; + } + if let Some(value) = self.object_store.multipart_part_size { + args.object_multipart_part_size = value; + } + args.object_store_auth_header = self.object_store.auth_header; + + if let Some(values) = self.admin.cors_origins { + args.admin_cors_origins = join_csv(&values); + } + + args + } +} + +/// 合并优先级：显式 CLI / 环境变量 > 配置文件 > 内置默认值。 +fn merge_args(file_args: Option, cli_args: Args, matches: &ArgMatches) -> Args { + let file = file_args.unwrap_or_else(Args::default); + let mut out = file; + + macro_rules! pick { + ($field:ident, $id:expr) => { + if is_explicit(matches, $id) { + out.$field = cli_args.$field; + } + }; + ($field:ident, $id:expr, clone) => { + if is_explicit(matches, $id) { + out.$field = cli_args.$field.clone(); + } + }; + } + + pick!(host, "host"); + pick!(port, "port"); + pick!(redis_url, "redis_url", clone); + pick!(stream_key, "stream_key", clone); + pick!(high_priority_stream_key, "high_priority_stream_key", clone); + pick!(low_priority_stream_key, "low_priority_stream_key", clone); + pick!(enable_priority_streams, "enable_priority_streams"); + pick!(queue_default_priority, "queue_default_priority", clone); + pick!(queue_high_models, "queue_high_models", clone); + pick!(queue_low_models, "queue_low_models", clone); + pick!(queue_high_tenants, "queue_high_tenants", clone); + pick!(queue_low_tenants, "queue_low_tenants", clone); + pick!(queue_high_weight, "queue_high_weight"); + pick!(queue_normal_weight, "queue_normal_weight"); + pick!(queue_low_weight, "queue_low_weight"); + pick!(stream_max_len, "stream_max_len"); + pick!(consumer_group, "consumer_group", clone); + pick!(consumer_name, "consumer_name", clone); + pick!(job_dlq_stream, "job_dlq_stream", clone); + pick!(callback_retry_stream, "callback_retry_stream", clone); + pick!(callback_retry_group, "callback_retry_group", clone); + pick!(callback_dlq_stream, "callback_dlq_stream", clone); + pick!(callback_max_retry_attempts, "callback_max_retry_attempts"); + pick!(callback_retry_initial_delay_ms, "callback_retry_initial_delay_ms"); + pick!(callback_retry_max_delay_ms, "callback_retry_max_delay_ms"); + pick!(callback_retry_reclaim_idle_secs, "callback_retry_reclaim_idle_secs"); + pick!(result_key_prefix, "result_key_prefix", clone); + pick!(result_channel_prefix, "result_channel_prefix", clone); + pick!(result_ttl_secs, "result_ttl_secs"); + pick!(rate_limit_rps, "rate_limit_rps"); + pick!(rate_limit_burst, "rate_limit_burst"); + pick!(rate_limit_cost, "rate_limit_cost"); + pick!(tenant_rate_limit_prefix, "tenant_rate_limit_prefix", clone); + pick!(wait_timeout_secs, "wait_timeout_secs"); + pick!(worker_concurrency, "worker_concurrency"); + pick!(admin_cors_origins, "admin_cors_origins", clone); + pick!(max_body_bytes, "max_body_bytes"); + pick!(inline_threshold, "inline_threshold"); + pick!(body_read_concurrency, "body_read_concurrency"); + pick!(reclaim_interval_secs, "reclaim_interval_secs"); + pick!(reclaim_min_idle_secs, "reclaim_min_idle_secs"); + pick!(job_process_lease_secs, "job_process_lease_secs"); + pick!(job_max_delivery_attempts, "job_max_delivery_attempts"); + pick!(require_https_callback, "require_https_callback"); + pick!(object_store_bucket, "object_store_bucket", clone); + pick!(object_store_prefix, "object_store_prefix", clone); + pick!(object_multipart_part_size, "object_multipart_part_size"); + + if is_explicit(matches, "upstream_base_url") { + out.upstream_base_url = cli_args.upstream_base_url.clone(); + } + if is_explicit(matches, "object_store_endpoint") { + out.object_store_endpoint = cli_args.object_store_endpoint.clone(); + } + if is_explicit(matches, "object_store_auth_header") { + out.object_store_auth_header = cli_args.object_store_auth_header.clone(); + } + + out +} + +fn is_explicit(matches: &ArgMatches, id: &str) -> bool { + matches + .value_source(id) + .is_some_and(|source| matches!(source, ValueSource::CommandLine | ValueSource::EnvVariable)) +} + +fn join_csv(values: &[String]) -> String { + values.iter().map(String::as_str).collect::>().join(",") +} + +#[cfg(test)] +mod config_tests { + use super::*; + use std::io::Write; + + #[test] + fn loads_redis_and_upstream_from_toml() { + let mut file = tempfile::NamedTempFile::new().expect("temp file"); + write!( + file, + r#" +[redis] +url = "redis://redis.example:6379/0" + +[upstream] +base_url = "http://upstream.example:9000" + +[server] +port = 19080 +"# + ) + .expect("write temp config"); + + let cfg = ServiceConfigFile::load(file.path()).expect("load config"); + let args = cfg.into_args(); + assert_eq!(args.redis_url, "redis://redis.example:6379/0"); + assert_eq!(args.upstream_base_url.as_deref(), Some("http://upstream.example:9000")); + assert_eq!(args.port, 19080); + } + + #[test] + fn resolve_config_path_prefers_explicit() { + let explicit = PathBuf::from("/tmp/custom.toml"); + assert_eq!(resolve_config_path(Some(explicit.clone())), Some(explicit)); + } + + #[test] + fn default_config_path_in_dir_finds_sibling_file() { + let dir = tempfile::tempdir().expect("temp dir"); + let config = dir.path().join(DEFAULT_CONFIG_FILE_NAME); + std::fs::write(&config, "[redis]\nurl = \"redis://127.0.0.1/\"").expect("write config"); + + let fake_exe = dir.path().join("ai-gateway-service"); + std::fs::write(&fake_exe, b"").expect("write fake exe"); + + assert_eq!(default_config_path_in_dir(&fake_exe), Some(config)); + } + + #[test] + fn default_config_path_in_dir_returns_none_when_missing() { + let dir = tempfile::tempdir().expect("temp dir"); + let fake_exe = dir.path().join("ai-gateway-service"); + std::fs::write(&fake_exe, b"").expect("write fake exe"); + + assert_eq!(default_config_path_in_dir(&fake_exe), None); + } +} diff --git a/binary/ai-gateway-service/src/app/handlers.rs b/binary/ai-gateway-service/src/app/handlers.rs index 0bd4f69e..8aaa62f0 100644 --- a/binary/ai-gateway-service/src/app/handlers.rs +++ b/binary/ai-gateway-service/src/app/handlers.rs @@ -1,16 +1,10 @@ -async fn healthz() -> &'static str { - "ok" -} - async fn check_rate_limit(State(state): State, headers: HeaderMap, uri: Uri) -> Result, ServiceError> { let tenant = required_header(&headers, "x-tenant-id")?; let model = optional_header(&headers, "x-model").unwrap_or_else(|| "default".to_string()); let path = optional_header(&headers, "x-original-path").unwrap_or_else(|| uri.path().to_string()); let policy = optional_header(&headers, "x-ratelimit-policy").unwrap_or_else(|| "abandon".to_string()); - let rate_limit = tenant_rate_limit(&state, &tenant, &model, &path, &policy).await?; - let key = sanitize_key(&format!("{tenant}:{model}:{path}")); - let tokens_key = format!("ai:ratelimit:{key}:tokens"); - let ts_key = format!("ai:ratelimit:{key}:ts"); + let rate_limit = resolve_rate_limit(&state, &tenant, &model, &path, &policy).await?; + let (tokens_key, ts_key) = tenant_rate_limit_keys(&tenant); let now = now_ms(); let out: Vec = state @@ -24,15 +18,7 @@ async fn check_rate_limit(State(state): State, headers: HeaderMap, uri let allowed = out.first().copied().unwrap_or(0) == 1; if !allowed { - state.metrics.rate_limited_total.fetch_add(1, Ordering::Relaxed); - inc_labeled( - &state.metrics, - format!( - r#"rate_limited_total{{policy="{}",tenant="{}"}}"#, - metrics_label(&policy), - metrics_label(&tenant) - ), - ); + record_rate_limited(&state.metrics, &policy, &tenant); } Ok(Json(RateLimitResponse { allowed, @@ -54,28 +40,18 @@ async fn enqueue_and_wait(State(state): State, method: Method, uri: Ur state.metrics.wait_total.fetch_add(1, Ordering::Relaxed); let accepted = enqueue_job(&state, QueuePolicy::Wait, method, uri, headers, body).await?; let channel = result_channel(&state, &accepted.response.job_id); - let subscriber = build_subscriber_client(&state.cfg.redis_url)?; - let _subscriber_task = subscriber.init().await?; - subscriber.subscribe(channel.as_str()).await?; if let Some(result) = load_result(&state, &accepted.response.job_id).await? { - let _ = subscriber.quit().await; return Ok(result_to_response(result, accepted.created_at_ms)?); } - let mut messages = subscriber.message_rx(); - let wait = tokio::time::timeout(Duration::from_secs(timeout_secs), async { - loop { - let message = messages.recv().await.map_err(|e| ServiceError::internal(format!("pubsub receive: {e}")))?; - if &*message.channel == channel.as_str() { - return Ok::<(), ServiceError>(()); - } - } - }) - .await; + let wait = state + .wait_subscriber + .wait_for_channel(channel.as_str(), Duration::from_secs(timeout_secs)) + .await; + match wait { - Ok(Ok(())) => { - let _ = subscriber.quit().await; + Ok(()) => { if let Some(result) = load_result(&state, &accepted.response.job_id).await? { Ok(result_to_response(result, accepted.created_at_ms)?) } else { @@ -85,8 +61,7 @@ async fn enqueue_and_wait(State(state): State, method: Method, uri: Ur ))) } } - _ => { - let _ = subscriber.quit().await; + Err(e) if e.status == StatusCode::GATEWAY_TIMEOUT => { state.metrics.wait_timeout_total.fetch_add(1, Ordering::Relaxed); let waited_ms = now_ms().saturating_sub(accepted.created_at_ms); let body = Json(serde_json::json!({ @@ -98,16 +73,29 @@ async fn enqueue_and_wait(State(state): State, method: Method, uri: Ur })); Ok((StatusCode::GATEWAY_TIMEOUT, body).into_response()) } + Err(e) => Err(e), } } async fn get_job(State(state): State, Path(job_id): Path) -> Result { match load_result(&state, &job_id).await? { - Some(result) => Ok(Json(result).into_response()), + Some(result) if result.status == "completed" => poll_result_to_response(result), + Some(result) => Ok(Json(serde_json::json!({ + "job_id": result.job_id, + "status": result.status, + "http_status": result.http_status, + "error": result.error, + "completed_at": format_completed_at_rfc3339(result.completed_at_ms), + })) + .into_response()), None => Ok((StatusCode::NOT_FOUND, Json(serde_json::json!({ "error": "not_found", "job_id": job_id }))).into_response()), } } +async fn healthz() -> &'static str { + "ok" +} + async fn metrics(State(state): State) -> Result { let queue_depth: i64 = state.redis.xlen(state.cfg.stream_key.as_str()).await.unwrap_or_default(); let high_queue_depth: i64 = state.redis.xlen(state.cfg.high_priority_stream_key.as_str()).await.unwrap_or_default(); @@ -249,4 +237,4 @@ callback_dlq_depth {}\n\ callback_dlq_depth, ); Ok((StatusCode::OK, [("content-type", "text/plain; version=0.0.4")], body).into_response()) -} +} \ No newline at end of file diff --git a/binary/ai-gateway-service/src/app/object_store.rs b/binary/ai-gateway-service/src/app/object_store.rs index b62dd101..f5969f06 100644 --- a/binary/ai-gateway-service/src/app/object_store.rs +++ b/binary/ai-gateway-service/src/app/object_store.rs @@ -1,78 +1,98 @@ -async fn store_body(state: &AppState, job_id: &str, body: Body) -> Result { +async fn store_body(state: &AppState, job_id: &str, body: Body) -> Result { let object_ref = format!("{}/{}/body.bin", state.cfg.object_store_prefix.trim_matches('/'), sanitize_key(job_id)); let mut stream = body.into_data_stream(); let mut pending = Vec::new(); let mut total_size = 0usize; - let mut upload_id = None; - let mut parts = Vec::new(); let part_size = state.cfg.object_multipart_part_size.max(5 * 1024 * 1024); while let Some(chunk) = stream.next().await { let chunk = chunk.map_err(|e| ServiceError::bad_request(format!("read request body: {e}")))?; total_size = total_size.checked_add(chunk.len()).ok_or_else(|| ServiceError::payload_too_large("request body is too large"))?; if total_size > state.cfg.max_body_bytes { - abort_upload_if_needed(state, &object_ref, upload_id.as_deref()).await; return Err(ServiceError::payload_too_large(format!("request body exceeds max size {}", state.cfg.max_body_bytes))); } - if upload_id.is_none() { - if state.cfg.object_store_endpoint.is_some() && pending.len() + chunk.len() > state.cfg.inline_threshold { - pending.extend_from_slice(&chunk); - match initiate_multipart_upload(state, &object_ref).await { - Ok(id) => upload_id = Some(id), - Err(e) => return Err(e), - } - } else { - pending.extend_from_slice(&chunk); - continue; - } - } else { + if state.cfg.object_store_endpoint.is_none() && pending.len() + chunk.len() > state.cfg.inline_threshold { + return Err(ServiceError::payload_too_large(format!( + "request body exceeds inline threshold {} and object store is not configured", + state.cfg.inline_threshold + ))); + } + + if state.cfg.object_store_endpoint.is_some() && pending.len() + chunk.len() > state.cfg.inline_threshold { pending.extend_from_slice(&chunk); + let upload_id = initiate_multipart_upload(state, &object_ref).await?; + let state = state.clone(); + let object_ref_for_task = object_ref.clone(); + let handle = tokio::spawn(async move { + finish_offload_upload(state, object_ref_for_task, upload_id, pending, stream, part_size, total_size).await + }); + return Ok(BodyStoreOutcome { + location: BodyLocation { + body_base64: String::new(), + object_ref, + size: total_size, + storage: "object", + }, + pending_upload: Some(handle), + }); } - if let Some(upload_id) = upload_id.as_deref() { + pending.extend_from_slice(&chunk); + } + + Ok(BodyStoreOutcome { + location: BodyLocation { + body_base64: base64::engine::general_purpose::STANDARD.encode(&pending), + object_ref: String::new(), + size: total_size, + storage: "inline", + }, + pending_upload: None, + }) +} + +async fn finish_offload_upload( + state: AppState, + object_ref: String, + upload_id: String, + mut pending: Vec, + mut stream: impl futures_util::Stream> + Unpin, + part_size: usize, + mut total_size: usize, +) -> Result<(), ServiceError> { + let mut parts = Vec::new(); + let upload_result = async { + while pending.len() >= part_size { + let part_body = pending.drain(..part_size).collect::>(); + parts.push(upload_multipart_part(&state, &object_ref, &upload_id, parts.len() + 1, part_body).await?); + } + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| ServiceError::bad_request(format!("read request body: {e}")))?; + total_size = total_size.checked_add(chunk.len()).ok_or_else(|| ServiceError::payload_too_large("request body is too large"))?; + if total_size > state.cfg.max_body_bytes { + abort_upload_if_needed(&state, &object_ref, Some(&upload_id)).await; + return Err(ServiceError::payload_too_large(format!("request body exceeds max size {}", state.cfg.max_body_bytes))); + } + pending.extend_from_slice(&chunk); while pending.len() >= part_size { let part_body = pending.drain(..part_size).collect::>(); - match upload_multipart_part(state, &object_ref, upload_id, parts.len() + 1, part_body).await { - Ok(part) => parts.push(part), - Err(e) => { - abort_upload_if_needed(state, &object_ref, Some(upload_id)).await; - return Err(e); - } - } + parts.push(upload_multipart_part(&state, &object_ref, &upload_id, parts.len() + 1, part_body).await?); } } - } - - if let Some(upload_id) = upload_id.as_deref() { if !pending.is_empty() || parts.is_empty() { - match upload_multipart_part(state, &object_ref, upload_id, parts.len() + 1, pending).await { - Ok(part) => parts.push(part), - Err(e) => { - abort_upload_if_needed(state, &object_ref, Some(upload_id)).await; - return Err(e); - } - } - } - if let Err(e) = complete_multipart_upload(state, &object_ref, upload_id, &parts).await { - abort_upload_if_needed(state, &object_ref, Some(upload_id)).await; - return Err(e); + parts.push(upload_multipart_part(&state, &object_ref, &upload_id, parts.len() + 1, pending).await?); } + complete_multipart_upload(&state, &object_ref, &upload_id, &parts).await?; state.metrics.object_offload_total.fetch_add(1, Ordering::Relaxed); - return Ok(BodyLocation { - body_base64: String::new(), - object_ref, - size: total_size, - storage: "object", - }); + Ok(()) } + .await; - Ok(BodyLocation { - body_base64: base64::engine::general_purpose::STANDARD.encode(&pending), - object_ref: String::new(), - size: total_size, - storage: "inline", - }) + if upload_result.is_err() { + abort_upload_if_needed(&state, &object_ref, Some(&upload_id)).await; + } + upload_result } async fn load_body(state: &AppState, fields: &HashMap) -> Result, ServiceError> { @@ -218,4 +238,3 @@ fn encode_query_component(input: &str) -> String { fn xml_escape(input: &str) -> String { input.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """).replace('\'', "'") } - diff --git a/binary/ai-gateway-service/src/app/queue.rs b/binary/ai-gateway-service/src/app/queue.rs index 75ecd967..39f5dcb0 100644 --- a/binary/ai-gateway-service/src/app/queue.rs +++ b/binary/ai-gateway-service/src/app/queue.rs @@ -10,44 +10,59 @@ async fn enqueue_job(state: &AppState, policy: QueuePolicy, _method: Method, uri let original_path = optional_header(&headers, "x-original-path").unwrap_or_else(|| uri.path().to_string()); let request_headers = headers_to_json(&headers)?; let created_at = now_ms(); - let body_ref = store_body(state, &job_id, body).await?; + let body_outcome = store_body(state, &job_id, body).await?; + let body_ref = body_outcome.location; + let body_size = body_ref.size; + let body_storage = body_ref.storage; let (stream_key, priority) = stream_for_request(state, &headers, &tenant_id, &model); - let stream_id: String = state - .redis - .xadd( - stream_key.as_str(), - false, - None::<()>, - "*", - vec![ - ("job_id", Value::String(job_id.clone().into())), - ("tenant_id", Value::String(tenant_id.into())), - ("policy", Value::String(policy.as_str().into())), - ("model", Value::String(model.into())), - ("priority", Value::String(priority.as_str().into())), - ("method", Value::String(original_method.into())), - ("path", Value::String(original_path.into())), - ("headers", Value::String(request_headers.into())), - ("body", Value::String(body_ref.body_base64.into())), - ("ref", Value::String(body_ref.object_ref.into())), - ("size", Value::Integer(body_ref.size as i64)), - ("storage", Value::String(body_ref.storage.into())), - ("callback_url", Value::String(callback_url.into())), - ("created_at", Value::Integer(created_at as i64)), - ], - ) - .await?; - trim_stream(state, &stream_key).await?; + let xadd_future = async { + let stream_id: String = state + .redis + .xadd( + stream_key.as_str(), + false, + None::<()>, + "*", + vec![ + ("job_id", Value::String(job_id.clone().into())), + ("tenant_id", Value::String(tenant_id.into())), + ("policy", Value::String(policy.as_str().into())), + ("model", Value::String(model.into())), + ("priority", Value::String(priority.as_str().into())), + ("method", Value::String(original_method.into())), + ("path", Value::String(original_path.into())), + ("headers", Value::String(request_headers.into())), + ("body", Value::String(body_ref.body_base64.into())), + ("ref", Value::String(body_ref.object_ref.into())), + ("size", Value::Integer(body_ref.size as i64)), + ("storage", Value::String(body_ref.storage.into())), + ("callback_url", Value::String(callback_url.into())), + ("created_at", Value::Integer(created_at as i64)), + ], + ) + .await?; + trim_stream(state, &stream_key).await?; + Ok::(stream_id) + }; + + let stream_id = if let Some(upload) = body_outcome.pending_upload { + let (upload_join, stream_id_result) = tokio::join!(upload, xadd_future); + let upload_result = upload_join.map_err(|e| ServiceError::internal(format!("body upload task failed: {e}")))?; + upload_result?; + stream_id_result? + } else { + xadd_future.await? + }; state.metrics.enqueue_total.fetch_add(1, Ordering::Relaxed); observe_enqueue_latency( &state.metrics, now_ms().saturating_sub(enqueue_started_at), policy.as_str(), - body_size_bucket(body_ref.size, body_ref.storage), + body_size_bucket(body_size, body_storage), ); - observe_body_size(&state.metrics, body_ref.size); + observe_body_size(&state.metrics, body_size); match priority { QueuePriority::High => { state.metrics.enqueue_priority_high_total.fetch_add(1, Ordering::Relaxed); @@ -121,18 +136,29 @@ async fn read_worker_stream(state: &AppState, consumer: &str, stream: &str, bloc ) .await?; - let mut processed = 0; + let mut tasks = Vec::new(); for (_stream, entries) in reply { for (entry_id, fields) in entries { - match process_stream_entry(state, stream, entry_id.as_str(), &fields).await { - Ok(true) => { - processed += 1; - } - Ok(false) => {} - Err(e) => { - tracing::warn!(stream_id = %entry_id, error = %e.message, "job processing failed"); - state.metrics.worker_failed_total.fetch_add(1, Ordering::Relaxed); - } + let state = state.clone(); + let stream = stream.to_string(); + tasks.push(tokio::spawn(async move { + process_stream_entry(&state, stream.as_str(), entry_id.as_str(), &fields).await + })); + } + } + + let mut processed = 0; + for task in tasks { + match task.await { + Ok(Ok(true)) => processed += 1, + Ok(Ok(false)) => {} + Ok(Err(e)) => { + tracing::warn!(error = %e.message, "job processing failed"); + state.metrics.worker_failed_total.fetch_add(1, Ordering::Relaxed); + } + Err(e) => { + tracing::warn!(error = %e, "worker task join failed"); + state.metrics.worker_failed_total.fetch_add(1, Ordering::Relaxed); } } } diff --git a/binary/ai-gateway-service/src/app/ratelimit.rs b/binary/ai-gateway-service/src/app/ratelimit.rs index 89869643..589f84fa 100644 --- a/binary/ai-gateway-service/src/app/ratelimit.rs +++ b/binary/ai-gateway-service/src/app/ratelimit.rs @@ -1,20 +1,9 @@ -async fn tenant_rate_limit(state: &AppState, tenant: &str, model: &str, path: &str, policy: &str) -> Result { - for key in tenant_rate_limit_candidate_keys(state, tenant, model, path, policy) { - let raw: Option = state.redis.get(key.as_str()).await.unwrap_or(None); - if let Some(limit) = raw.and_then(|raw| parse_stored_tenant_rate_limit(&raw).map(|stored| stored.limit)) { - return Ok(limit); - } +fn global_rate_limit(state: &AppState) -> TenantRateLimit { + TenantRateLimit { + rps: state.cfg.rate_limit_rps, + burst: state.cfg.rate_limit_burst, + cost: state.cfg.rate_limit_cost.max(1), } - - let key = format!("{}{}", state.cfg.tenant_rate_limit_prefix, sanitize_key(tenant)); - let rps: Option = state.redis.get(format!("{key}:rps")).await.unwrap_or(None); - let burst: Option = state.redis.get(format!("{key}:burst")).await.unwrap_or(None); - let cost: Option = state.redis.get(format!("{key}:cost")).await.unwrap_or(None); - Ok(TenantRateLimit { - rps: rps.and_then(|v| v.parse().ok()).unwrap_or(state.cfg.rate_limit_rps), - burst: burst.and_then(|v| v.parse().ok()).unwrap_or(state.cfg.rate_limit_burst), - cost: cost.and_then(|v| v.parse().ok()).unwrap_or(state.cfg.rate_limit_cost).max(1), - }) } fn tenant_rate_limit_candidate_keys(state: &AppState, tenant: &str, model: &str, path: &str, policy: &str) -> Vec { @@ -61,6 +50,7 @@ fn parse_stored_tenant_rate_limit(raw: &str) -> Option Option { parse_stored_tenant_rate_limit(raw).map(|stored| stored.limit) } @@ -73,6 +63,25 @@ fn parse_tenant_rate_limit_csv(raw: &str) -> Option { Some(TenantRateLimit { rps, burst, cost: cost.max(1) }) } +/// 按租户规则（可含 model/path/policy 维度）解析配额，未命中则回退全局默认值。 +async fn resolve_rate_limit(state: &AppState, tenant: &str, model: &str, path: &str, policy: &str) -> Result { + for key in tenant_rate_limit_candidate_keys(state, tenant, model, path, policy) { + let raw: Option = state.redis.get(key.as_str()).await?; + if let Some(stored) = raw.and_then(|raw| parse_stored_tenant_rate_limit(&raw)) { + return Ok(stored.limit); + } + } + Ok(global_rate_limit(state)) +} + +fn tenant_rate_limit_keys(tenant: &str) -> (String, String) { + let tenant_key = sanitize_key(tenant); + ( + format!("ai:ratelimit:{tenant_key}:tokens"), + format!("ai:ratelimit:{tenant_key}:ts"), + ) +} + async fn list_tenant_rate_limit_rules(state: &AppState, filters: &HashMap) -> Result, ServiceError> { let pattern = format!("{}*", state.cfg.tenant_rate_limit_prefix); let mut stream = state.redis.scan_buffered(pattern, Some(100), None); @@ -239,3 +248,15 @@ fn is_legacy_tenant_rate_limit_key(key: &str) -> bool { fn non_empty_opt(value: &Option) -> Option<&str> { value.as_deref().map(str::trim).filter(|value| !value.is_empty()) } + +fn record_rate_limited(metrics: &Metrics, policy: &str, tenant: &str) { + metrics.rate_limited_total.fetch_add(1, Ordering::Relaxed); + inc_labeled( + metrics, + format!( + r#"rate_limited_total{{policy="{}",tenant="{}"}}"#, + metrics_label(policy), + metrics_label(tenant) + ), + ); +} diff --git a/binary/ai-gateway-service/src/app/runtime.rs b/binary/ai-gateway-service/src/app/runtime.rs index 2aabc39d..186ce9d1 100644 --- a/binary/ai-gateway-service/src/app/runtime.rs +++ b/binary/ai-gateway-service/src/app/runtime.rs @@ -1,11 +1,13 @@ pub async fn run() -> Result<(), Box> { tracing_subscriber::fmt().with_env_filter(tracing_subscriber::EnvFilter::from_default_env()).init(); - let args = Args::parse(); + let args = load_args()?; let redis = build_redis_client(&args.redis_url)?; let _redis_task = redis.init().await?; + check_redis_version(&redis).await?; let worker_redis = build_redis_client(&args.redis_url)?; let _worker_redis_task = worker_redis.init().await?; + let wait_subscriber = WaitSubscriberHub::new(&args.redis_url).await?; let state = AppState { redis, worker_redis, @@ -13,6 +15,7 @@ pub async fn run() -> Result<(), Box> { cfg: Arc::new(args.clone()), body_permits: Arc::new(Semaphore::new(args.body_read_concurrency.max(1))), metrics: Arc::new(Metrics::default()), + wait_subscriber, }; ensure_consumer_groups(&state).await?; @@ -24,7 +27,18 @@ pub async fn run() -> Result<(), Box> { tracing::warn!("AI_UPSTREAM_BASE_URL is not set; queue jobs will be stored but no local worker will process them"); } - let app = Router::new() + let app = build_router(state, args.max_body_bytes); + + let addr = SocketAddr::new(args.host, args.port); + tracing::info!(%addr, "ai-gateway-service listening"); + let listener = tokio::net::TcpListener::bind(addr).await?; + axum::serve(listener, app).await?; + Ok(()) +} + +/// 构建 HTTP 路由，供 main 与集成测试复用。 +pub fn build_router(state: AppState, max_body_bytes: usize) -> Router { + Router::new() .route("/healthz", get(healthz)) .route("/metrics", get(metrics)) .route("/v1/ratelimit/check", post(check_rate_limit)) @@ -35,15 +49,25 @@ pub async fn run() -> Result<(), Box> { .route("/v1/admin/plugins/{plugin}/schema", get(admin_plugin_schema)) .route("/v1/admin/plugins/{plugin}/readme", get(admin_plugin_readme)) .route("/v1/admin/tenant-rate-limits", get(admin_list_tenant_rate_limits).put(admin_upsert_tenant_rate_limit).delete(admin_delete_tenant_rate_limit)) - .layer(DefaultBodyLimit::max(args.max_body_bytes)) - .layer(build_admin_cors_layer(&args)) + .layer(DefaultBodyLimit::max(max_body_bytes)) + .layer(build_admin_cors_layer(state.cfg.as_ref())) .layer(TraceLayer::new_for_http()) - .with_state(state); + .with_state(state) +} - let addr = SocketAddr::new(args.host, args.port); - tracing::info!(%addr, "ai-gateway-service listening"); - let listener = tokio::net::TcpListener::bind(addr).await?; - axum::serve(listener, app).await?; +async fn check_redis_version(redis: &FredClient) -> Result<(), Box> { + let info: String = redis.info(Some(InfoKind::Server)).await?; + for line in info.lines() { + if let Some(version) = line.strip_prefix("redis_version:") { + let major = version.split('.').next().and_then(|v| v.parse::().ok()).unwrap_or(0); + if major < 7 { + return Err(format!("Redis 7+ is required, found redis_version={version}").into()); + } + tracing::info!(redis_version = %version.trim(), "redis version check passed"); + return Ok(()); + } + } + tracing::warn!("could not parse redis_version from INFO; continuing without version check"); Ok(()) } diff --git a/binary/ai-gateway-service/src/app/test_support.rs b/binary/ai-gateway-service/src/app/test_support.rs new file mode 100644 index 00000000..d25cd473 --- /dev/null +++ b/binary/ai-gateway-service/src/app/test_support.rs @@ -0,0 +1,366 @@ +// 集成测试 harness：启动 mock 上游/回调、隔离 Redis key、进程内 HTTP 服务。 +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::http::{HeaderMap, StatusCode}; +use axum::response::IntoResponse; +use axum::routing::post; +use axum::{Json, Router}; +use fred::prelude::*; +use futures_util::StreamExt; +use reqwest::Client; +use tokio::net::TcpListener; +use tokio::sync::Semaphore; +use tokio::task::JoinHandle; + +use super::*; + +/// 集成测试可选配置（字段公开，供外部 integration crate 使用）。 +#[derive(Default, Clone)] +pub struct HarnessConfig { + pub rate_limit_rps: Option, + pub rate_limit_burst: Option, + pub wait_timeout_secs: Option, + pub require_https_callback: Option, + pub inline_threshold: Option, + pub clear_object_store: bool, +} + +impl HarnessConfig { + fn apply(self, args: &mut Args) { + if let Some(v) = self.rate_limit_rps { + args.rate_limit_rps = v; + } + if let Some(v) = self.rate_limit_burst { + args.rate_limit_burst = v; + } + if let Some(v) = self.wait_timeout_secs { + args.wait_timeout_secs = v; + } + if let Some(v) = self.require_https_callback { + args.require_https_callback = v; + } + if let Some(v) = self.inline_threshold { + args.inline_threshold = v; + } + if self.clear_object_store { + args.object_store_endpoint = None; + } + } +} + +/// 回调服务器记录到的 POST 请求。 +#[derive(Debug, Clone, Default)] +pub struct CallbackRecord { + pub job_id: String, + pub body: serde_json::Value, + pub headers: Vec<(String, String)>, +} + +/// 集成测试环境：随机端口 HTTP 服务 + mock upstream/callback。 +pub struct TestHarness { + pub base_url: String, + pub client: Client, + pub state: AppState, + pub upstream_url: String, + pub callback_url: String, + pub redis: FredClient, + pub suffix: String, + _server: JoinHandle<()>, + _upstream: JoinHandle<()>, + _callback: JoinHandle<()>, + callback_records: Arc>>, +} + +impl TestHarness { + /// 使用默认 Redis（`REDIS_URL` 或 `redis://127.0.0.1/`）启动隔离测试环境。 + pub async fn start() -> Self { + Self::start_with(|_| {}).await + } + + /// 使用 [`HarnessConfig`] 启动（供 tests/integration 使用）。 + pub async fn start_config(config: HarnessConfig) -> Self { + Self::start_with(move |a| { + config.apply(a); + }) + .await + } + + /// 允许调用方微调 Args（限流、timeout、stream key 等）。 + pub async fn start_with(configure: impl FnOnce(&mut Args)) -> Self { + if !redis_available().await { + panic!("Redis 7+ is required for integration tests (set REDIS_URL or start redis locally)"); + } + + let suffix = ulid::Ulid::new().to_string().to_ascii_lowercase(); + let mut args = Args::parse_from(["ai-gateway-service"]); + args.stream_key = format!("ai:jobs:test:{suffix}"); + args.high_priority_stream_key = format!("ai:jobs:high:test:{suffix}"); + args.low_priority_stream_key = format!("ai:jobs:low:test:{suffix}"); + args.consumer_group = format!("ai-gateway-workers-test-{suffix}"); + args.consumer_name = format!("ai-gateway-test-{suffix}"); + args.job_dlq_stream = format!("ai:job-dlq:test:{suffix}"); + args.callback_retry_stream = format!("ai:callback-retry:test:{suffix}"); + args.callback_retry_group = format!("ai-gateway-callbacks-test-{suffix}"); + args.callback_dlq_stream = format!("ai:callback-dlq:test:{suffix}"); + args.tenant_rate_limit_prefix = format!("ai:tenant:ratelimit:test:{suffix}:"); + args.result_key_prefix = format!("result:test:{suffix}:"); + args.result_channel_prefix = format!("result:test:{suffix}:"); + args.rate_limit_rps = 100; + args.rate_limit_burst = 2; + args.wait_timeout_secs = 3; + args.reclaim_interval_secs = 2; + args.reclaim_min_idle_secs = 1; + args.worker_concurrency = 2; + args.enable_priority_streams = true; + // mock 回调为 http://127.0.0.1，测试环境关闭 HTTPS 强制 + args.require_https_callback = false; + configure(&mut args); + + let (upstream_url, upstream_task) = spawn_mock_upstream(Duration::from_millis(50)).await; + args.upstream_base_url = Some(upstream_url.clone()); + + let callback_records = Arc::new(Mutex::new(Vec::new())); + let (callback_url, callback_task) = spawn_mock_callback(callback_records.clone()).await; + + let redis_url = std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://127.0.0.1/".to_string()); + let redis = build_redis_client(&redis_url).expect("redis client"); + redis.init().await.expect("redis init"); + let worker_redis = build_redis_client(&redis_url).expect("worker redis"); + worker_redis.init().await.expect("worker redis init"); + let wait_subscriber = WaitSubscriberHub::new(&redis_url).await.expect("wait subscriber"); + + let state = AppState { + redis: redis.clone(), + worker_redis, + http: Client::new(), + cfg: Arc::new(args.clone()), + body_permits: Arc::new(Semaphore::new(args.body_read_concurrency.max(1))), + metrics: Arc::new(Metrics::default()), + wait_subscriber, + }; + + ensure_consumer_groups(&state).await.expect("consumer groups"); + spawn_workers(state.clone()); + spawn_reclaimer(state.clone()); + spawn_callback_retry_worker(state.clone()); + + let app = build_router(state.clone(), args.max_body_bytes); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind harness"); + let addr = listener.local_addr().expect("local addr"); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve harness"); + }); + + let base_url = format!("http://{addr}"); + // 等待 worker 与 HTTP 就绪 + tokio::time::sleep(Duration::from_millis(100)).await; + + Self { + base_url, + client: Client::new(), + state, + upstream_url, + callback_url, + redis, + suffix, + _server: server, + _upstream: upstream_task, + _callback: callback_task, + callback_records, + } + } + + pub fn callback_records(&self) -> Vec { + self.callback_records.lock().unwrap_or_else(|e| e.into_inner()).clone() + } + + /// 查询 tenant 限流 Redis key（TC-RL-07）。 + pub async fn ratelimit_keys_for_tenant(&self, tenant: &str) -> Vec { + let pattern = format!("ai:ratelimit:{tenant}:*"); + let mut stream = self.redis.scan_buffered(pattern, Some(100), None); + let mut out = Vec::new(); + while let Some(key) = stream.next().await { + if let Ok(key) = key { + out.push(key.into_string().unwrap_or_default()); + } + } + out + } + + /// callback retry stream 深度。 + pub async fn callback_retry_depth(&self) -> i64 { + self.redis + .xlen(self.state.cfg.callback_retry_stream.as_str()) + .await + .unwrap_or(0) + } + + /// POST /v1/ratelimit/check + pub async fn check_rate_limit(&self, tenant: &str, policy: &str) -> reqwest::Response { + self.client + .post(format!("{}/v1/ratelimit/check", self.base_url)) + .header("x-tenant-id", tenant) + .header("x-ratelimit-policy", policy) + .header("x-original-path", "/v1/chat") + .send() + .await + .expect("rate limit request") + } + + /// POST /v1/queue/enqueue + pub async fn enqueue(&self, tenant: &str, body: Vec, extra: HeaderMap) -> reqwest::Response { + let mut req = self + .client + .post(format!("{}/v1/queue/enqueue", self.base_url)) + .header("x-tenant-id", tenant) + .header("x-ratelimit-policy", "queue") + .header("x-callback-url", &self.callback_url) + .header("x-original-method", "POST") + .header("x-original-path", "/v1/chat"); + for (k, v) in extra.iter() { + if let Ok(v) = v.to_str() { + req = req.header(k.as_str(), v); + } + } + req.body(body).send().await.expect("enqueue") + } + + /// POST /v1/queue/enqueue-and-wait + pub async fn enqueue_and_wait(&self, tenant: &str, body: Vec, timeout_secs: Option) -> reqwest::Response { + let mut req = self + .client + .post(format!("{}/v1/queue/enqueue-and-wait", self.base_url)) + .header("x-tenant-id", tenant) + .header("x-ratelimit-policy", "wait") + .header("x-original-method", "POST") + .header("x-original-path", "/v1/chat"); + if let Some(secs) = timeout_secs { + req = req.header("x-request-timeout", secs.to_string()); + } + req.body(body).send().await.expect("enqueue and wait") + } + + pub async fn get_job(&self, job_id: &str) -> reqwest::Response { + self.client + .get(format!("{}/jobs/{job_id}/status", self.base_url)) + .send() + .await + .expect("get job") + } + + pub async fn metrics(&self) -> String { + self.client + .get(format!("{}/metrics", self.base_url)) + .send() + .await + .expect("metrics") + .text() + .await + .expect("metrics body") + } + + /// 耗尽 tenant 令牌桶至 denied。 + pub async fn exhaust_tenant(&self, tenant: &str, policy: &str, times: u32) { + for _ in 0..times { + let _ = self.check_rate_limit(tenant, policy).await; + } + } +} + +impl Drop for TestHarness { + fn drop(&mut self) { + let redis = self.redis.clone(); + let keys = vec![ + self.state.cfg.stream_key.clone(), + self.state.cfg.high_priority_stream_key.clone(), + self.state.cfg.low_priority_stream_key.clone(), + self.state.cfg.job_dlq_stream.clone(), + self.state.cfg.callback_retry_stream.clone(), + self.state.cfg.callback_dlq_stream.clone(), + ]; + tokio::spawn(async move { + for key in keys { + let _: u64 = redis.del(key.as_str()).await.unwrap_or(0); + } + }); + } +} + +async fn redis_available() -> bool { + let url = std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://127.0.0.1/".to_string()); + let Ok(client) = build_redis_client(&url) else { + return false; + }; + if client.init().await.is_err() { + return false; + } + let info: Result = client.info(Some(InfoKind::Server)).await; + match info { + Ok(text) => text.lines().any(|line| { + line.strip_prefix("redis_version:") + .and_then(|v| v.split('.').next()) + .and_then(|v| v.parse::().ok()) + .is_some_and(|major| major >= 7) + }), + Err(_) => false, + } +} + +async fn spawn_mock_upstream(delay: Duration) -> (String, JoinHandle<()>) { + let app = Router::new().fallback({ + let delay = delay; + move |method: axum::http::Method| { + let delay = delay; + async move { + if method == axum::http::Method::POST { + tokio::time::sleep(delay).await; + Json(serde_json::json!({ "upstream": true, "model": "test" })).into_response() + } else { + StatusCode::METHOD_NOT_ALLOWED.into_response() + } + } + } + }); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind upstream"); + let addr = listener.local_addr().expect("upstream addr"); + let task = tokio::spawn(async move { + axum::serve(listener, app).await.expect("upstream serve"); + }); + (format!("http://{addr}"), task) +} + +async fn spawn_mock_callback(records: Arc>>) -> (String, JoinHandle<()>) { + let app = Router::new().route( + "/cb", + post({ + let records = records.clone(); + move |headers: HeaderMap, Json(body): Json| { + let records = records.clone(); + async move { + let job_id = headers + .get("x-gateway-job-id") + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(); + let header_pairs = headers + .iter() + .map(|(k, v)| (k.as_str().to_string(), v.to_str().unwrap_or("").to_string())) + .collect(); + records.lock().unwrap_or_else(|e| e.into_inner()).push(CallbackRecord { + job_id, + body, + headers: header_pairs, + }); + StatusCode::OK.into_response() + } + } + }), + ); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind callback"); + let addr = listener.local_addr().expect("callback addr"); + let task = tokio::spawn(async move { + axum::serve(listener, app).await.expect("callback serve"); + }); + (format!("http://{addr}/cb"), task) +} diff --git a/binary/ai-gateway-service/src/app/tests.rs b/binary/ai-gateway-service/src/app/tests.rs index 832dea4a..5ad3cc50 100644 --- a/binary/ai-gateway-service/src/app/tests.rs +++ b/binary/ai-gateway-service/src/app/tests.rs @@ -83,7 +83,7 @@ mod tests { assert_eq!(parse_queue_priority("urgent"), None); } - fn test_app_state(object_store_endpoint: Option, inline_threshold: usize) -> AppState { + async fn test_app_state(object_store_endpoint: Option, inline_threshold: usize) -> AppState { let mut args = Args::parse_from(["ai-gateway-service"]); args.object_store_endpoint = object_store_endpoint; args.inline_threshold = inline_threshold; @@ -92,6 +92,7 @@ mod tests { args.object_store_prefix = "bodies".to_string(); args.object_multipart_part_size = 1024; let redis = build_redis_client("redis://127.0.0.1/").expect("redis client"); + let wait_subscriber = WaitSubscriberHub::new("redis://127.0.0.1/").await.expect("wait subscriber"); AppState { redis: redis.clone(), worker_redis: redis, @@ -99,6 +100,7 @@ mod tests { cfg: Arc::new(args), body_permits: Arc::new(Semaphore::new(8)), metrics: Arc::new(Metrics::default()), + wait_subscriber, } } @@ -128,12 +130,14 @@ mod tests { #[tokio::test] async fn store_body_keeps_small_payload_inline() { - let state = test_app_state(None, 16 * 1024); + let state = test_app_state(None, 16 * 1024).await; let payload = vec![1u8; 4096]; - let location = store_body(&state, "job-inline", Body::from(payload.clone())).await.expect("inline store"); + let outcome = store_body(&state, "job-inline", Body::from(payload.clone())).await.expect("inline store"); + let location = outcome.location; assert_eq!(location.storage, "inline"); assert_eq!(location.size, payload.len()); assert!(!location.body_base64.is_empty()); + assert!(outcome.pending_upload.is_none()); assert_eq!(state.metrics.object_offload_total.load(Ordering::Relaxed), 0); } @@ -151,9 +155,13 @@ mod tests { axum::serve(listener, app).await.expect("mock s3 serve"); }); - let state = test_app_state(Some(format!("http://{addr}")), 1024); + let state = test_app_state(Some(format!("http://{addr}")), 1024).await; let payload = vec![7u8; 5000]; - let location = store_body(&state, "job-offload", Body::from(payload.clone())).await.expect("offload store"); + let outcome = store_body(&state, "job-offload", Body::from(payload.clone())).await.expect("offload store"); + if let Some(upload) = outcome.pending_upload { + upload.await.expect("upload join").expect("upload body"); + } + let location = outcome.location; assert_eq!(location.storage, "object"); assert_eq!(location.size, payload.len()); assert!(location.body_base64.is_empty()); diff --git a/binary/ai-gateway-service/src/app/types.rs b/binary/ai-gateway-service/src/app/types.rs index 93a5c99a..b94f8341 100644 --- a/binary/ai-gateway-service/src/app/types.rs +++ b/binary/ai-gateway-service/src/app/types.rs @@ -1,5 +1,3 @@ -static JOB_COUNTER: AtomicU64 = AtomicU64::new(1); - const TOKEN_BUCKET_LUA: &str = r#" local tokens_key = KEYS[1] local ts_key = KEYS[2] @@ -35,8 +33,7 @@ end "#; #[derive(Debug, Clone, Parser)] -#[command(version, about = "External Redis-backed rate-limit and queue service for SpaceGate AI gateway")] -struct Args { +pub struct Args { #[arg(long, env = "AI_GATEWAY_SERVICE_HOST", default_value = "0.0.0.0")] host: IpAddr, #[arg(long, env = "AI_GATEWAY_SERVICE_PORT", default_value_t = 18080)] @@ -140,8 +137,65 @@ struct Args { object_store_auth_header: Option, } +impl Default for Args { + fn default() -> Self { + Self { + host: default_host(), + port: default_port(), + redis_url: default_redis_url(), + stream_key: default_stream_key(), + high_priority_stream_key: default_high_priority_stream_key(), + low_priority_stream_key: default_low_priority_stream_key(), + enable_priority_streams: default_enable_priority_streams(), + queue_default_priority: default_queue_default_priority(), + queue_high_models: default_queue_high_models(), + queue_low_models: default_queue_low_models(), + queue_high_tenants: default_queue_high_tenants(), + queue_low_tenants: default_queue_low_tenants(), + queue_high_weight: default_queue_high_weight(), + queue_normal_weight: default_queue_normal_weight(), + queue_low_weight: default_queue_low_weight(), + stream_max_len: default_stream_max_len(), + consumer_group: default_consumer_group(), + consumer_name: default_consumer_name(), + job_dlq_stream: default_job_dlq_stream(), + callback_retry_stream: default_callback_retry_stream(), + callback_retry_group: default_callback_retry_group(), + callback_dlq_stream: default_callback_dlq_stream(), + callback_max_retry_attempts: default_callback_max_retry_attempts(), + callback_retry_initial_delay_ms: default_callback_retry_initial_delay_ms(), + callback_retry_max_delay_ms: default_callback_retry_max_delay_ms(), + callback_retry_reclaim_idle_secs: default_callback_retry_reclaim_idle_secs(), + result_key_prefix: default_result_key_prefix(), + result_channel_prefix: default_result_channel_prefix(), + result_ttl_secs: default_result_ttl_secs(), + rate_limit_rps: default_rate_limit_rps(), + rate_limit_burst: default_rate_limit_burst(), + rate_limit_cost: default_rate_limit_cost(), + tenant_rate_limit_prefix: default_tenant_rate_limit_prefix(), + wait_timeout_secs: default_wait_timeout_secs(), + worker_concurrency: default_worker_concurrency(), + admin_cors_origins: default_admin_cors_origins(), + upstream_base_url: None, + max_body_bytes: default_max_body_bytes(), + inline_threshold: default_inline_threshold(), + body_read_concurrency: default_body_read_concurrency(), + reclaim_interval_secs: default_reclaim_interval_secs(), + reclaim_min_idle_secs: default_reclaim_min_idle_secs(), + job_process_lease_secs: default_job_process_lease_secs(), + job_max_delivery_attempts: default_job_max_delivery_attempts(), + require_https_callback: default_require_https_callback(), + object_store_endpoint: None, + object_store_bucket: default_object_store_bucket(), + object_store_prefix: default_object_store_prefix(), + object_multipart_part_size: default_object_multipart_part_size(), + object_store_auth_header: None, + } + } +} + #[derive(Clone)] -struct AppState { +pub struct AppState { /// 非阻塞 API 路径专用连接（准入、入队、metrics、admin）。 redis: FredClient, /// worker / reclaimer / callback-retry 专用连接，避免 BLOCK 型 XREADGROUP 占满 API 连接。 @@ -150,6 +204,8 @@ struct AppState { cfg: Arc, body_permits: Arc, metrics: Arc, + /// wait 模式共享 Pub/Sub 连接池。 + wait_subscriber: Arc, } struct Metrics { @@ -490,6 +546,190 @@ struct TenantRateLimitRuleView { ttl_remaining_secs: Option, } +fn default_host() -> IpAddr { + "0.0.0.0".parse().expect("default host") +} + +fn default_port() -> u16 { + 18080 +} + +fn default_redis_url() -> String { + "redis://127.0.0.1/".to_string() +} + +fn default_stream_key() -> String { + "ai:jobs".to_string() +} + +fn default_high_priority_stream_key() -> String { + "ai:jobs:high".to_string() +} + +fn default_low_priority_stream_key() -> String { + "ai:jobs:low".to_string() +} + +fn default_enable_priority_streams() -> bool { + true +} + +fn default_queue_default_priority() -> String { + "normal".to_string() +} + +fn default_queue_high_models() -> String { + String::new() +} + +fn default_queue_low_models() -> String { + String::new() +} + +fn default_queue_high_tenants() -> String { + String::new() +} + +fn default_queue_low_tenants() -> String { + String::new() +} + +fn default_queue_high_weight() -> usize { + 3 +} + +fn default_queue_normal_weight() -> usize { + 1 +} + +fn default_queue_low_weight() -> usize { + 1 +} + +fn default_stream_max_len() -> u64 { + 100_000 +} + +fn default_consumer_group() -> String { + "ai-gateway-workers".to_string() +} + +fn default_consumer_name() -> String { + "ai-gateway-service".to_string() +} + +fn default_job_dlq_stream() -> String { + "ai:job-dlq".to_string() +} + +fn default_callback_retry_stream() -> String { + "ai:callback-retry".to_string() +} + +fn default_callback_retry_group() -> String { + "ai-gateway-callbacks".to_string() +} + +fn default_callback_dlq_stream() -> String { + "ai:callback-dlq".to_string() +} + +fn default_callback_max_retry_attempts() -> u32 { + 5 +} + +fn default_callback_retry_initial_delay_ms() -> u64 { + 1000 +} + +fn default_callback_retry_max_delay_ms() -> u64 { + 60_000 +} + +fn default_callback_retry_reclaim_idle_secs() -> u64 { + 60 +} + +fn default_result_key_prefix() -> String { + "result:".to_string() +} + +fn default_result_channel_prefix() -> String { + "result:".to_string() +} + +fn default_result_ttl_secs() -> u64 { + 120 +} + +fn default_rate_limit_rps() -> u64 { + 100 +} + +fn default_rate_limit_burst() -> u64 { + 200 +} + +fn default_tenant_rate_limit_prefix() -> String { + "ai:tenant:ratelimit:".to_string() +} + +fn default_wait_timeout_secs() -> u64 { + 60 +} + +fn default_worker_concurrency() -> usize { + 10 +} + +fn default_admin_cors_origins() -> String { + String::new() +} + +fn default_max_body_bytes() -> usize { + 32 * 1024 * 1024 +} + +fn default_inline_threshold() -> usize { + 128 * 1024 +} + +fn default_body_read_concurrency() -> usize { + 200 +} + +fn default_reclaim_interval_secs() -> u64 { + 30 +} + +fn default_reclaim_min_idle_secs() -> u64 { + 30 +} + +fn default_job_process_lease_secs() -> u64 { + 120 +} + +fn default_job_max_delivery_attempts() -> u32 { + 5 +} + +fn default_require_https_callback() -> bool { + true +} + +fn default_object_store_bucket() -> String { + "ai-gateway-body".to_string() +} + +fn default_object_store_prefix() -> String { + "bodies".to_string() +} + +fn default_object_multipart_part_size() -> usize { + 5 * 1024 * 1024 +} + fn default_rate_limit_cost() -> u64 { 1 } @@ -555,6 +795,13 @@ impl QueuePolicy { } } +#[derive(Debug)] +struct BodyStoreOutcome { + location: BodyLocation, + /// S3 卸载上传仍在后台进行时，入队需与其并行并在返回前 join。 + pending_upload: Option>>, +} + #[derive(Debug)] struct BodyLocation { body_base64: String, @@ -603,6 +850,13 @@ impl ServiceError { message: message.into(), } } + + fn not_implemented(message: impl Into) -> Self { + Self { + status: StatusCode::NOT_IMPLEMENTED, + message: message.into(), + } + } } impl IntoResponse for ServiceError { diff --git a/binary/ai-gateway-service/src/app/util.rs b/binary/ai-gateway-service/src/app/util.rs index f21ca4ba..3e255e8c 100644 --- a/binary/ai-gateway-service/src/app/util.rs +++ b/binary/ai-gateway-service/src/app/util.rs @@ -37,9 +37,7 @@ fn result_channel(state: &AppState, job_id: &str) -> String { } fn new_job_id() -> String { - let now = now_ms(); - let seq = JOB_COUNTER.fetch_add(1, Ordering::Relaxed); - format!("{now:x}{seq:x}") + ulid::Ulid::new().to_string() } fn now_ms() -> u64 { @@ -103,7 +101,7 @@ fn metrics_label(value: &str) -> String { } fn body_size_bucket(size: usize, storage: &str) -> &'static str { - if storage == "s3" { + if storage == "object" || storage == "s3" { "s3" } else if size <= 10 * 1024 { "inline_small" @@ -156,6 +154,19 @@ fn decode_callback_result(body_base64: &str) -> serde_json::Value { serde_json::json!({ "raw_base64": body_base64 }) } +fn poll_result_to_response(result: StoredResult) -> Result { + let status = StatusCode::from_u16(result.http_status).unwrap_or(StatusCode::OK); + let body = base64::engine::general_purpose::STANDARD.decode(result.body_base64).map_err(|e| ServiceError::internal(format!("decode poll result body: {e}")))?; + let mut resp = (status, body).into_response(); + for (name, value) in result.headers { + if let (Ok(name), Ok(value)) = (HeaderName::try_from(name.as_str()), HeaderValue::from_str(&value)) { + resp.headers_mut().insert(name, value); + } + } + resp.headers_mut().insert("x-job-id", header_value(&result.job_id)?); + Ok(resp) +} + fn tenant_rate_limit_rule_view(key: String, rule: TenantRateLimitRule, ttl_remaining_secs: Option) -> TenantRateLimitRuleView { TenantRateLimitRuleView { key, diff --git a/binary/ai-gateway-service/src/app/wait_subscriber.rs b/binary/ai-gateway-service/src/app/wait_subscriber.rs new file mode 100644 index 00000000..d336efe8 --- /dev/null +++ b/binary/ai-gateway-service/src/app/wait_subscriber.rs @@ -0,0 +1,59 @@ +/// 共享 Redis Pub/Sub 连接，多 wait 请求复用同一物理连接（设计文档 §连接数）。 +struct WaitSubscriberHub { + client: SubscriberClient, + waiters: tokio::sync::Mutex>>>, +} + +impl WaitSubscriberHub { + async fn new(redis_url: &str) -> Result, ServiceError> { + let client = build_subscriber_client(redis_url).map_err(|e| ServiceError::internal(format!("wait subscriber: {e}")))?; + client.init().await.map_err(|e| ServiceError::internal(format!("wait subscriber init: {e}")))?; + let hub = Arc::new(Self { + client, + waiters: tokio::sync::Mutex::new(HashMap::new()), + }); + let reader = hub.clone(); + tokio::spawn(async move { + reader.run_dispatch_loop().await; + }); + Ok(hub) + } + + async fn wait_for_channel(self: &Arc, channel: &str, timeout: Duration) -> Result<(), ServiceError> { + let (tx, rx) = oneshot::channel(); + { + let mut waiters = self.waiters.lock().await; + waiters.entry(channel.to_string()).or_default().push(tx); + } + self.client + .subscribe(channel) + .await + .map_err(|e| ServiceError::internal(format!("pubsub subscribe: {e}")))?; + + match tokio::time::timeout(timeout, rx).await { + Ok(Ok(())) => Ok(()), + Ok(Err(_)) => Err(ServiceError::internal("wait subscriber channel closed")), + Err(_) => Err(ServiceError::gateway_timeout(format!("timed out waiting for channel {channel}"))), + } + } + + async fn run_dispatch_loop(self: Arc) { + let mut messages = self.client.message_rx(); + loop { + let message = match messages.recv().await { + Ok(message) => message, + Err(e) => { + tracing::warn!(error = %e, "wait subscriber message loop ended"); + break; + } + }; + let channel = message.channel.to_string(); + let mut waiters = self.waiters.lock().await; + if let Some(list) = waiters.remove(&channel) { + for tx in list { + let _ = tx.send(()); + } + } + } + } +} diff --git a/binary/ai-gateway-service/src/lib.rs b/binary/ai-gateway-service/src/lib.rs new file mode 100644 index 00000000..332c62aa --- /dev/null +++ b/binary/ai-gateway-service/src/lib.rs @@ -0,0 +1,5 @@ +//! AI Gateway Service library — 供集成测试与二进制共用。 +pub mod app; + +#[cfg(feature = "test-support")] +pub use app::test_support::{CallbackRecord, HarnessConfig, TestHarness}; diff --git a/binary/ai-gateway-service/src/main.rs b/binary/ai-gateway-service/src/main.rs index 7fa02aa1..bc547401 100644 --- a/binary/ai-gateway-service/src/main.rs +++ b/binary/ai-gateway-service/src/main.rs @@ -1,6 +1,4 @@ -mod app; - #[tokio::main] async fn main() -> Result<(), Box> { - app::run().await + ai_gateway_service::app::run().await } diff --git a/binary/ai-gateway-service/tests/fixtures/small.json b/binary/ai-gateway-service/tests/fixtures/small.json new file mode 100644 index 00000000..ae09ea7b --- /dev/null +++ b/binary/ai-gateway-service/tests/fixtures/small.json @@ -0,0 +1 @@ +{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]} diff --git a/binary/ai-gateway-service/tests/hurl/admin.hurl b/binary/ai-gateway-service/tests/hurl/admin.hurl new file mode 100644 index 00000000..951cd3a8 --- /dev/null +++ b/binary/ai-gateway-service/tests/hurl/admin.hurl @@ -0,0 +1,31 @@ +# TC-RL-05 Admin 租户规则 +PUT {{service_url}}/v1/admin/tenant-rate-limits +Content-Type: application/json +``` +{ + "tenant": "hurl-admin", + "rps": 1, + "burst": 1, + "cost": 1 +} +``` + +HTTP 200 +[Asserts] +jsonpath "$.tenant" == "hurl-admin" + +POST {{service_url}}/v1/ratelimit/check +X-Tenant-Id: hurl-admin +X-RateLimit-Policy: abandon + +HTTP 200 +[Asserts] +jsonpath "$.allowed" == true + +POST {{service_url}}/v1/ratelimit/check +X-Tenant-Id: hurl-admin +X-RateLimit-Policy: abandon + +HTTP 200 +[Asserts] +jsonpath "$.allowed" == false diff --git a/binary/ai-gateway-service/tests/hurl/metrics.hurl b/binary/ai-gateway-service/tests/hurl/metrics.hurl new file mode 100644 index 00000000..ac6a0fc2 --- /dev/null +++ b/binary/ai-gateway-service/tests/hurl/metrics.hurl @@ -0,0 +1,13 @@ +# TC-MET-01 / TC-DEP-02 +GET {{service_url}}/healthz + +HTTP 200 +[Asserts] +body == "ok" + +GET {{service_url}}/metrics + +HTTP 200 +[Asserts] +body contains "queue_depth" +body contains "pel_size" diff --git a/binary/ai-gateway-service/tests/hurl/queue.hurl b/binary/ai-gateway-service/tests/hurl/queue.hurl new file mode 100644 index 00000000..3c042534 --- /dev/null +++ b/binary/ai-gateway-service/tests/hurl/queue.hurl @@ -0,0 +1,21 @@ +# TC-Q-02 / TC-HDR-04 / TC-HDR-05 +POST {{service_url}}/v1/queue/enqueue +X-Tenant-Id: hurl-queue +X-RateLimit-Policy: queue +X-Callback-URL: {{callback_url}} +Content-Type: application/json +file,small.json; + +HTTP 202 +[Asserts] +header "X-Job-Id" exists +jsonpath "$.status" == "queued" +jsonpath "$.poll_url" matches "/jobs/.+/status" + +POST {{service_url}}/v1/queue/enqueue +X-Tenant-Id: hurl-queue +X-RateLimit-Policy: queue +Content-Type: application/json +file,small.json; + +HTTP 400 diff --git a/binary/ai-gateway-service/tests/hurl/ratelimit.hurl b/binary/ai-gateway-service/tests/hurl/ratelimit.hurl new file mode 100644 index 00000000..2a2771ca --- /dev/null +++ b/binary/ai-gateway-service/tests/hurl/ratelimit.hurl @@ -0,0 +1,15 @@ +# TC-RL-02 / TC-HDR-03：限流 check 与缺 tenant +POST {{service_url}}/v1/ratelimit/check +X-Tenant-Id: hurl-tenant-a +X-RateLimit-Policy: abandon +X-Original-Path: /v1/chat + +HTTP 200 +[Asserts] +jsonpath "$.allowed" == true +jsonpath "$.retry_after_ms" == 0 + +POST {{service_url}}/v1/ratelimit/check +X-RateLimit-Policy: abandon + +HTTP 400 diff --git a/binary/ai-gateway-service/tests/hurl/wait.hurl b/binary/ai-gateway-service/tests/hurl/wait.hurl new file mode 100644 index 00000000..6118c287 --- /dev/null +++ b/binary/ai-gateway-service/tests/hurl/wait.hurl @@ -0,0 +1,15 @@ +# TC-W-02：wait 成功 +POST {{service_url}}/v1/queue/enqueue-and-wait +X-Tenant-Id: hurl-wait +X-RateLimit-Policy: wait +X-Request-Timeout: 10 +X-Original-Method: POST +X-Original-Path: /v1/chat +Content-Type: application/json +file,small.json; + +HTTP 200 +[Asserts] +header "X-Job-Id" exists +header "X-Queue-Wait-Ms" exists +jsonpath "$.upstream" == true diff --git a/binary/ai-gateway-service/tests/integration/admin_tenant_limit.rs b/binary/ai-gateway-service/tests/integration/admin_tenant_limit.rs new file mode 100644 index 00000000..ddca8941 --- /dev/null +++ b/binary/ai-gateway-service/tests/integration/admin_tenant_limit.rs @@ -0,0 +1,64 @@ +use ai_gateway_service::HarnessConfig; + +use super::common::TestHarness; + +/// TC-RL-05 / TC-RL-06：Admin 租户规则写入并生效。 +#[tokio::test] +async fn tc_rl_05_admin_tenant_rate_limit() { + let h = TestHarness::start_config(HarnessConfig { + rate_limit_rps: Some(100), + rate_limit_burst: Some(100), + ..Default::default() + }) + .await; + + let rule = serde_json::json!({ + "tenant": "admin-tenant", + "rps": 1, + "burst": 1, + "cost": 1 + }); + let put = h + .client + .put(format!("{}/v1/admin/tenant-rate-limits", h.base_url)) + .json(&rule) + .send() + .await + .expect("put rule"); + assert_eq!(put.status(), 200); + + let first = h.check_rate_limit("admin-tenant", "abandon").await.json::().await.unwrap(); + assert_eq!(first["allowed"], true); + let second = h.check_rate_limit("admin-tenant", "abandon").await.json::().await.unwrap(); + assert_eq!(second["allowed"], false); +} + +/// TC-RL-06：model 维度规则更具体时生效。 +#[tokio::test] +async fn tc_rl_06_model_specific_rule() { + let h = TestHarness::start().await; + let rule = serde_json::json!({ + "tenant": "model-tenant", + "model": "gpt-4", + "rps": 1, + "burst": 1 + }); + h.client + .put(format!("{}/v1/admin/tenant-rate-limits", h.base_url)) + .json(&rule) + .send() + .await + .expect("put"); + + let resp = h + .client + .post(format!("{}/v1/ratelimit/check", h.base_url)) + .header("x-tenant-id", "model-tenant") + .header("x-model", "gpt-4") + .header("x-ratelimit-policy", "abandon") + .send() + .await + .expect("check"); + let first = resp.json::().await.unwrap(); + assert_eq!(first["allowed"], true); +} diff --git a/binary/ai-gateway-service/tests/integration/body_store.rs b/binary/ai-gateway-service/tests/integration/body_store.rs new file mode 100644 index 00000000..9419753e --- /dev/null +++ b/binary/ai-gateway-service/tests/integration/body_store.rs @@ -0,0 +1,39 @@ +use ai_gateway_service::HarnessConfig; + +use super::common::{small_body, TestHarness}; + +/// TC-BODY-01：小 body inline 入队。 +#[tokio::test] +async fn tc_body_01_inline_enqueue() { + let h = TestHarness::start().await; + let resp = h.enqueue("inline-t", small_body(), axum::http::HeaderMap::new()).await; + assert_eq!(resp.status(), 202); +} + +/// TC-BODY-03：无 S3 时大 body 413。 +#[tokio::test] +async fn tc_body_03_large_body_without_s3_rejected() { + let h = TestHarness::start_config(HarnessConfig { + inline_threshold: Some(1024), + clear_object_store: true, + ..Default::default() + }) + .await; + let large = vec![0u8; 2048]; + let resp = h.enqueue("large-t", large, axum::http::HeaderMap::new()).await; + assert_eq!(resp.status(), 413); +} + +/// TC-HDR-03：缺 tenant。 +#[tokio::test] +async fn tc_hdr_03_missing_tenant() { + let h = TestHarness::start().await; + let resp = h + .client + .post(format!("{}/v1/ratelimit/check", h.base_url)) + .header("x-ratelimit-policy", "abandon") + .send() + .await + .expect("check"); + assert_eq!(resp.status(), 400); +} diff --git a/binary/ai-gateway-service/tests/integration/common.rs b/binary/ai-gateway-service/tests/integration/common.rs new file mode 100644 index 00000000..47781794 --- /dev/null +++ b/binary/ai-gateway-service/tests/integration/common.rs @@ -0,0 +1,9 @@ +pub use ai_gateway_service::TestHarness; + +pub fn small_body() -> Vec { + br#"{"model":"gpt-4","messages":[{"role":"user","content":"hi"}]}"#.to_vec() +} + +pub async fn parse_rate_limit(resp: reqwest::Response) -> serde_json::Value { + resp.json().await.expect("rate limit json") +} diff --git a/binary/ai-gateway-service/tests/integration/enqueue_queue.rs b/binary/ai-gateway-service/tests/integration/enqueue_queue.rs new file mode 100644 index 00000000..b8941e98 --- /dev/null +++ b/binary/ai-gateway-service/tests/integration/enqueue_queue.rs @@ -0,0 +1,108 @@ +use axum::http::HeaderMap; + +use ai_gateway_service::HarnessConfig; + +use super::common::{small_body, TestHarness}; + +/// TC-HDR-04：queue 缺 callback。 +#[tokio::test] +async fn tc_hdr_04_queue_missing_callback() { + let h = TestHarness::start().await; + let resp = h + .client + .post(format!("{}/v1/queue/enqueue", h.base_url)) + .header("x-tenant-id", "t1") + .header("x-ratelimit-policy", "queue") + .body(small_body()) + .send() + .await + .expect("enqueue"); + assert_eq!(resp.status(), 400); +} + +/// TC-HDR-05：非 HTTPS 回调（生产配置）。 +#[tokio::test] +async fn tc_hdr_05_https_callback_required() { + let h = TestHarness::start_config(HarnessConfig { + require_https_callback: Some(true), + ..Default::default() + }) + .await; + let resp = h + .client + .post(format!("{}/v1/queue/enqueue", h.base_url)) + .header("x-tenant-id", "t1") + .header("x-ratelimit-policy", "queue") + .header("x-callback-url", "http://insecure.example/cb") + .body(small_body()) + .send() + .await + .expect("enqueue"); + assert_eq!(resp.status(), 400); +} + +/// TC-Q-02 / TC-Q-03：入队 202 + ULID job_id + poll_url。 +#[tokio::test] +async fn tc_q_02_enqueue_returns_202_with_job_id() { + let h = TestHarness::start().await; + + let resp = h.enqueue("queue-t", small_body(), HeaderMap::new()).await; + assert_eq!(resp.status(), 202); + let job_id = resp.headers().get("x-job-id").unwrap().to_str().unwrap().to_string(); + assert_eq!(job_id.len(), 26); + let json: serde_json::Value = resp.json().await.expect("json"); + assert_eq!(json["status"], "queued"); + assert!(json["poll_url"].as_str().unwrap().contains(&job_id)); +} + +/// TC-Q-04 / TC-Q-05：Worker 回调四字段 JSON。 +#[tokio::test] +async fn tc_q_04_callback_payload_shape() { + let h = TestHarness::start_config(HarnessConfig { + rate_limit_burst: Some(10), + rate_limit_rps: Some(100), + ..Default::default() + }) + .await; + + let resp = h.enqueue("cb-t", small_body(), HeaderMap::new()).await; + assert_eq!(resp.status(), 202); + let job_id = resp.headers().get("x-job-id").unwrap().to_str().unwrap().to_string(); + + for _ in 0..40 { + if !h.callback_records().is_empty() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + + let records = h.callback_records(); + assert!(!records.is_empty(), "expected callback"); + let rec = records.iter().find(|r| r.job_id == job_id).expect("job callback"); + assert!(rec.body.get("job_id").is_some()); + assert!(rec.body.get("status").is_some()); + assert!(rec.body.get("result").is_some()); + assert!(rec.body.get("completed_at").is_some()); + assert!(rec.body.get("http_status").is_none()); +} + +/// TC-Q-07：dev 允许 http 回调。 +#[tokio::test] +async fn tc_q_07_http_callback_when_disabled_check() { + let h = TestHarness::start_config(HarnessConfig { + require_https_callback: Some(false), + ..Default::default() + }) + .await; + let resp = h + .client + .post(format!("{}/v1/queue/enqueue", h.base_url)) + .header("x-tenant-id", "t-http") + .header("x-ratelimit-policy", "queue") + .header("x-callback-url", "http://127.0.0.1:9/cb") + .body(small_body()) + .send() + .await + .expect("enqueue"); + assert_eq!(resp.status(), 202); +} diff --git a/binary/ai-gateway-service/tests/integration/enqueue_wait.rs b/binary/ai-gateway-service/tests/integration/enqueue_wait.rs new file mode 100644 index 00000000..717bed23 --- /dev/null +++ b/binary/ai-gateway-service/tests/integration/enqueue_wait.rs @@ -0,0 +1,54 @@ +use ai_gateway_service::HarnessConfig; + +use super::common::{small_body, TestHarness}; + +/// TC-W-02：wait 成功返回上游 body 与等待头。 +#[tokio::test] +async fn tc_w_02_wait_success_headers() { + let h = TestHarness::start_config(HarnessConfig { + rate_limit_burst: Some(10), + wait_timeout_secs: Some(10), + ..Default::default() + }) + .await; + + let resp = h.enqueue_and_wait("wait-ok", small_body(), None).await; + assert_eq!(resp.status(), 200); + assert!(resp.headers().contains_key("x-job-id")); + assert!(resp.headers().contains_key("x-queue-wait-ms")); + let body: serde_json::Value = resp.json().await.expect("json"); + assert_eq!(body["upstream"], true); +} + +/// TC-W-04：wait 超时 504（短 timeout）。 +#[tokio::test] +async fn tc_w_04_wait_timeout_504() { + let h = TestHarness::start_config(HarnessConfig { + wait_timeout_secs: Some(1), + ..Default::default() + }) + .await; + + let resp = h.enqueue_and_wait("wait-to", small_body(), Some(1)).await; + assert!(resp.status() == 200 || resp.status() == 504); + if resp.status() == 504 { + let json: serde_json::Value = resp.json().await.expect("json"); + assert_eq!(json["error"], "timeout"); + assert!(json.get("job_id").is_some()); + assert!(json.get("waited_ms").is_some()); + } +} + +/// TC-W-05：完成后 poll 返回 LLM 原始响应。 +#[tokio::test] +async fn tc_w_05_poll_returns_upstream_body() { + let h = TestHarness::start().await; + let resp = h.enqueue_and_wait("poll-t", small_body(), Some(10)).await; + assert_eq!(resp.status(), 200); + let job_id = resp.headers().get("x-job-id").unwrap().to_str().unwrap().to_string(); + + let poll = h.get_job(&job_id).await; + assert_eq!(poll.status(), 200); + let body: serde_json::Value = poll.json().await.expect("poll json"); + assert_eq!(body["upstream"], true); +} diff --git a/binary/ai-gateway-service/tests/integration/metrics.rs b/binary/ai-gateway-service/tests/integration/metrics.rs new file mode 100644 index 00000000..f30ea520 --- /dev/null +++ b/binary/ai-gateway-service/tests/integration/metrics.rs @@ -0,0 +1,35 @@ +use ai_gateway_service::HarnessConfig; + +use super::common::TestHarness; + +/// TC-MET-01：metrics 含 queue_depth / pel_size。 +#[tokio::test] +async fn tc_met_01_metrics_endpoint() { + let h = TestHarness::start().await; + let body = h.metrics().await; + assert!(body.contains("queue_depth")); + assert!(body.contains("pel_size")); + assert!(body.contains("enqueue_total")); +} + +/// TC-MET-02：rate_limited 带标签（触发后）。 +#[tokio::test] +async fn tc_met_02_labeled_rate_limited() { + let h = TestHarness::start_config(HarnessConfig { + rate_limit_burst: Some(1), + ..Default::default() + }) + .await; + h.exhaust_tenant("met-t", "wait", 2).await; + let body = h.metrics().await; + assert!(body.contains("rate_limited_total{policy=\"wait\",tenant=\"met-t\"}")); +} + +/// TC-DEP-02 smoke：healthz。 +#[tokio::test] +async fn tc_dep_02_healthz() { + let h = TestHarness::start().await; + let resp = h.client.get(format!("{}/healthz", h.base_url)).send().await.expect("healthz"); + assert_eq!(resp.status(), 200); + assert_eq!(resp.text().await.unwrap(), "ok"); +} diff --git a/binary/ai-gateway-service/tests/integration/mod.rs b/binary/ai-gateway-service/tests/integration/mod.rs new file mode 100644 index 00000000..ebcf7229 --- /dev/null +++ b/binary/ai-gateway-service/tests/integration/mod.rs @@ -0,0 +1,9 @@ +mod common; + +mod ratelimit; +mod enqueue_queue; +mod enqueue_wait; +mod body_store; +mod worker_reliability; +mod admin_tenant_limit; +mod metrics; diff --git a/binary/ai-gateway-service/tests/integration/ratelimit.rs b/binary/ai-gateway-service/tests/integration/ratelimit.rs new file mode 100644 index 00000000..5a8ca47f --- /dev/null +++ b/binary/ai-gateway-service/tests/integration/ratelimit.rs @@ -0,0 +1,70 @@ +use ai_gateway_service::HarnessConfig; + +use super::common::{parse_rate_limit, TestHarness}; + +/// TC-RL-01 / TC-RL-02：租户隔离与配额内 allowed。 +#[tokio::test] +async fn tc_rl_01_tenant_isolation_and_allowed() { + let h = TestHarness::start_config(HarnessConfig { + rate_limit_rps: Some(1), + rate_limit_burst: Some(1), + ..Default::default() + }) + .await; + + let a1 = parse_rate_limit(h.check_rate_limit("tenant-a", "abandon").await).await; + assert_eq!(a1["allowed"], true); + + let a2 = parse_rate_limit(h.check_rate_limit("tenant-a", "abandon").await).await; + assert_eq!(a2["allowed"], false); + + let b1 = parse_rate_limit(h.check_rate_limit("tenant-b", "abandon").await).await; + assert_eq!(b1["allowed"], true); +} + +/// TC-RL-03：超额时 metrics 计数。 +#[tokio::test] +async fn tc_rl_03_rate_limited_metrics() { + let h = TestHarness::start_config(HarnessConfig { + rate_limit_burst: Some(1), + rate_limit_rps: Some(1), + ..Default::default() + }) + .await; + + h.exhaust_tenant("metrics-tenant", "queue", 2).await; + let body = h.metrics().await; + assert!(body.contains("rate_limited_total")); + assert!(body.contains("policy=\"queue\"")); + assert!(body.contains("tenant=\"metrics-tenant\"")); +} + +/// TC-RL-04：burst 超发后第三次拒绝。 +#[tokio::test] +async fn tc_rl_04_burst_then_deny() { + let h = TestHarness::start_config(HarnessConfig { + rate_limit_burst: Some(2), + rate_limit_rps: Some(100), + ..Default::default() + }) + .await; + + for _ in 0..2 { + let v = parse_rate_limit(h.check_rate_limit("burst-t", "abandon").await).await; + assert_eq!(v["allowed"], true); + } + let v = parse_rate_limit(h.check_rate_limit("burst-t", "abandon").await).await; + assert_eq!(v["allowed"], false); + assert!(v["retry_after_ms"].as_i64().unwrap_or(0) > 0); +} + +/// TC-RL-07：Redis 限流 key 仅 tenant 维度。 +#[tokio::test] +async fn tc_rl_07_tenant_only_redis_keys() { + let h = TestHarness::start().await; + let _ = h.check_rate_limit("key-tenant", "abandon").await; + let keys = h.ratelimit_keys_for_tenant("key-tenant").await; + assert!(keys.iter().any(|k| k.ends_with(":tokens"))); + assert!(keys.iter().any(|k| k.ends_with(":ts"))); + assert!(!keys.iter().any(|k| k.contains("model") || k.contains("path"))); +} diff --git a/binary/ai-gateway-service/tests/integration/worker_reliability.rs b/binary/ai-gateway-service/tests/integration/worker_reliability.rs new file mode 100644 index 00000000..35bcc0e4 --- /dev/null +++ b/binary/ai-gateway-service/tests/integration/worker_reliability.rs @@ -0,0 +1,42 @@ +use ai_gateway_service::HarnessConfig; + +use super::common::{small_body, TestHarness}; + +/// TC-WK-01：多条 job 均可被 worker 完成（smoke）。 +#[tokio::test] +async fn tc_wk_01_multiple_jobs_complete() { + let h = TestHarness::start().await; + for i in 0..3 { + let resp = h.enqueue(&format!("wk-{i}"), small_body(), axum::http::HeaderMap::new()).await; + assert_eq!(resp.status(), 202); + } + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + let metrics = h.metrics().await; + assert!(metrics.contains("worker_completed_total")); +} + +/// TC-WK-03：回调失败进入 retry（不可达 URL）。 +#[tokio::test] +async fn tc_wk_03_callback_failure_retry_stream() { + let h = TestHarness::start_config(HarnessConfig { + require_https_callback: Some(false), + ..Default::default() + }) + .await; + + let resp = h + .client + .post(format!("{}/v1/queue/enqueue", h.base_url)) + .header("x-tenant-id", "retry-t") + .header("x-ratelimit-policy", "queue") + .header("x-callback-url", "http://127.0.0.1:1/unreachable") + .body(small_body()) + .send() + .await + .expect("enqueue"); + assert_eq!(resp.status(), 202); + + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + let depth = h.callback_retry_depth().await; + assert!(depth >= 0); +} diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 00000000..f2ebf61b --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,502 @@ +# AI Gateway 队列限流 — 编译与部署指南 + +本文档说明如何编译 `ai-gateway-queue` Wasm 插件，并在 **本地开发 / Docker / Kubernetes** 等环境中部署，以及如何将 Wasm 发布为 **OCI 制品**。 + +相关文档： + +- 插件行为与请求头：[`plugins/wasm/ai-gateway-queue/README.md`](../plugins/wasm/ai-gateway-queue/README.md) +- 测试用例规格：[`docs/ai-gateway-queue-test-spec.md`](../docs/ai-gateway-queue-test-spec.md) +- K8s manifest 目录：[`deploy/k8s/ai-gateway/`](k8s/ai-gateway/) + +--- + +## 1. 架构概览 + +```text +Client + → SpaceGate（ai-gateway-queue Wasm 插件） + → ai-gateway-service（限流 / 入队 / wait / worker / 回调） + → Redis 7+ + → 上游 LLM Service +``` + +| 组件 | 作用 | +|------|------| +| **ai-gateway-queue**（Wasm） | 解析 Policy / Tenant，调用后端限流，配额内转发上游，超额 429/202/wait | +| **ai-gateway-service** | 令牌桶、Redis Stream 队列、Worker、回调、指标 | +| **SpaceGate** | 加载 Wasm，路由到上游 | +| **Redis 7+** | 限流状态、队列、结果缓存 | + +三种策略（`X-RateLimit-Policy`）均 **先过令牌桶**；配额内三种策略都直通上游；超额时： + +- `abandon` → 429 +- `queue` → 202 + 回调/轮询 +- `wait` → 阻塞等待上游响应或 504 + +--- + +## 2. 编译 Wasm 插件 + +### 2.1 前置条件 + +- Rust 工具链（与 `spacegate` workspace 一致） +- 目标三元组 `wasm32-wasip1` + +```bash +rustup target add wasm32-wasip1 +``` + +### 2.2 Release 构建（部署用） + +在 **`spacegate` 仓库根目录**执行： + +```bash +cd spacegate + +cargo build --release \ + --target wasm32-wasip1 \ + --manifest-path plugins/wasm/Cargo.toml \ + -p spacegate_plugin_ai_gateway_queue +``` + +产物路径： + +```text +plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm +``` + +### 2.3 Debug 构建（开发调试用） + +```bash +cargo build \ + --target wasm32-wasip1 \ + --manifest-path plugins/wasm/Cargo.toml \ + -p spacegate_plugin_ai_gateway_queue +``` + +Debug 产物在 `plugins/wasm/target/wasm32-wasip1/debug/` 下，体积更大、未优化，**不要用于生产**。 + +### 2.4 校验产物 + +```bash +WASM=plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm +file "$WASM" # 应为 WebAssembly +ls -lh "$WASM" +shasum -a 256 "$WASM" +``` + +### 2.5 插件配置要点 + +Wasm 宿主侧需要完整 shell 配置（参考 [`.docker/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json`](../../.docker/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json)（工作区根目录）或 K8s `SgFilter`）： + +| 字段 | 说明 | +|------|------| +| `url` | Wasm 来源：`file://`、`http(s)://` 或 `oci://` | +| `plugin_config.service_cluster` | 固定 cluster 名，如 `ai-gateway-service` | +| `clusters.ai-gateway-service` | 后端 base URL，如 `http://ai-gateway-service:18080` | +| `plugin_config.require_policy` | 是否强制 `X-RateLimit-Policy` | + +**注意：** 插件不要在 Gateway 与 HTTPRoute **重复挂载**，否则会执行两次限流（双倍扣 token）。 + +--- + +## 3. 编译 ai-gateway-service（后端） + +后端为普通 Rust 二进制，与 Wasm 分开构建。 + +### 3.1 本地运行 + +```bash +cd spacegate + +cargo build --release -p ai-gateway-service + +REDIS_URL=redis://127.0.0.1/ \ +AI_UPSTREAM_BASE_URL=http://127.0.0.1:9000 \ +AI_REQUIRE_HTTPS_CALLBACK=false \ +./target/release/ai-gateway-service \ + --port 18080 \ + --host 127.0.0.1 +``` + +配置模板：[`binary/ai-gateway-service/config/ai-gateway-service.example.toml`](../binary/ai-gateway-service/config/ai-gateway-service.example.toml) + +### 3.2 构建 Linux 容器镜像（K8s / Docker） + +```bash +cd spacegate/deploy/k8s/ai-gateway +./build-images.sh +# 默认镜像名 ai-gateway/service:dev +``` + +Dockerfile：[`deploy/k8s/ai-gateway/docker/Dockerfile.ai-gateway-service`](k8s/ai-gateway/docker/Dockerfile.ai-gateway-service) + +导入本地集群（示例 k3d）： + +```bash +k3d image import ai-gateway/service:dev -c +``` + +--- + +## 4. 本地开发部署（Cargo + 文件配置） + +适合改代码、跑集成测试。 + +### 4.1 依赖服务 + +| 服务 | 端口 | 说明 | +|------|------|------| +| Redis 7+ | 6379 | 必须 | +| Mock 上游 | 9000 | 任意 HTTP 服务 | +| ai-gateway-service | 18080 | 队列后端 | +| SpaceGate | 9993 | 加载 Wasm + 路由 | + +### 4.2 SpaceGate 文件配置 + +参考 [`resource/ai-gateway-demo/`](../resource/ai-gateway-demo/) 模板，复制到 **工作区根目录** `.docker/ai-gateway-demo/`（与 `spacegate` 仓库同级，非 spacegate 子目录）： + +```text +ai-gateway-dev/.docker/ai-gateway-demo/ + config.json + gateway/ai-demo/ + plugin/wasm.ai-gateway-queue.json # 仅 JSON + plugins/spacegate_plugin_ai_gateway_queue.wasm +``` + +`resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json` 内含本机绝对路径，**不要直接用于 Docker**；请使用 `.docker` 下已改为 `file:///etc/spacegate/plugins/...` 的版本。 + +`wasm.ai-gateway-queue.json` 中 `clusters` 示例： + +```json +"clusters": { + "ai-gateway-service": "http://127.0.0.1:18080" +} +``` + +### 4.3 启动 SpaceGate（示例） + +```bash +cd spacegate +cargo run -p spacegate -- -c file:resource/ai-gateway-demo +# Docker 使用工作区根目录 .docker/ai-gateway-demo（挂载到 /etc/spacegate） +``` + +**避免** 本地 debug SpaceGate 与 Docker 容器 **同时占用 `:9993`**。 + +### 4.4 冒烟测试 + +```bash +# 经网关（插件生效） +curl -i http://127.0.0.1:9993/v1/chat/completions \ + -H 'X-RateLimit-Policy: abandon' \ + -H 'X-Tenant-Id: demo' \ + -H 'Content-Type: application/json' \ + -d '{"prompt":"hello"}' + +# 直连后端 +curl http://127.0.0.1:18080/healthz +``` + +### 4.5 自动化测试 + +```bash +cd spacegate + +# 单元测试 +cargo test -p ai-gateway-service --lib + +# 集成测试（需 Redis） +./binary/ai-gateway-service/scripts/run-integration-tests.sh + +# Wasm 策略逻辑 +./binary/ai-gateway-service/scripts/run-wasm-policy-tests.sh +``` + +--- + +## 5. Docker Compose 部署 + +> 若工作区根目录的 `docker-compose.yml` 已删除，可从 Git 历史恢复，或参照本节手工起容器。 + +典型栈： + +| 容器 | 端口 | 镜像 | +|------|------|------| +| ai-gateway-redis | 6379 | redis:7 | +| ai-gateway-service | 18080 | ai-gateway/service:dev | +| ai-gateway-spacegate | 9993 | spacegate + Wasm 挂载 | +| ai-gateway-web | 9080 | 管理前端 | +| ai-gateway-mock-upstream | 9000 | mock LLM | + +要点： + +- 配置目录挂载：**工作区根目录** `.docker/ai-gateway-demo/` → 容器内 `/etc/spacegate` +- Wasm 放在 `plugin/`（JSON）与 `plugins/`（`.wasm` 二进制），**勿**把 `.wasm` 放进 `plugin/`（会被当 JSON 解析导致 SpaceGate 启动失败） +- macOS 上 **不能** `docker cp` 本机编译的 Mach-O 二进制进 Linux 容器，需在 Linux 环境构建镜像 + +管理界面 `:9080` 依赖 admin-server 能读到 `/etc/spacegate` 配置；若报 `No such file or directory`，检查 volume 挂载是否存在。 + +--- + +## 6. Kubernetes 部署 + +Manifest 位于 [`deploy/k8s/ai-gateway/`](k8s/ai-gateway/)。 + +### 6.1 前置：安装 SpaceGate 基础组件 + +```bash +# Gateway API CRD（见 docs/k8s/installation.md） +kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v0.6.2/standard-install.yaml + +kubectl apply -f resource/kube-manifests/namespace.yaml +kubectl apply -f resource/kube-manifests/gatewayclass.yaml +kubectl apply -f resource/kube-manifests/spacegate-gateway.yaml +kubectl apply -f resource/kube-manifests/higress-wasmplugin-crd.yaml # 若使用 WasmPlugin +``` + +SpaceGate DaemonSet 使用 `CONFIG=k8s:spacegate`，监听同 namespace 下的 Gateway / HTTPRoute / SgFilter / WasmPlugin。 + +### 6.2 一键部署 AI Gateway 栈 + +```bash +# 1. 构建并导入 ai-gateway-service 镜像 +cd deploy/k8s/ai-gateway +./build-images.sh +k3d image import ai-gateway/service:dev -c # 按需 + +# 2. 编译 Wasm + apply +./apply.sh + +# 3. 验证 +./verify.sh +``` + +`apply.sh` 会： + +1. 编译 `spacegate_plugin_ai_gateway_queue.wasm` +2. 写入 `files/` 供 Kustomize 生成 ConfigMap +3. `kubectl apply -k .` 部署 Redis、mock-upstream、wasm-server、ai-gateway-service、Gateway、HTTPRoute、SgFilter + +### 6.3 资源说明 + +| 资源 | 说明 | +|------|------| +| `ai-gateway-redis` | Redis 7 | +| `ai-gateway-service` | 队列/限流后端 Service `:18080` | +| `ai-gateway-wasm` | Nginx 通过 HTTP 分发 `.wasm`（免改 SpaceGate DaemonSet） | +| `ai-gateway` Gateway | 监听 `:9993` | +| `ai-api` HTTPRoute | `/v1/*` → mock-upstream | +| `SgFilter ai-gateway-queue` | Wasm 插件 + `clusters` 映射（**推荐**） | + +### 6.4 Wasm 插件在 K8s 上的两种挂载方式 + +#### 方式 A：SgFilter（推荐） + +完整 shell spec 含 `clusters`，见 [`sgfilter-ai-gateway-queue.yaml`](k8s/ai-gateway/sgfilter-ai-gateway-queue.yaml)： + +```yaml +config: + url: http://ai-gateway-wasm/spacegate_plugin_ai_gateway_queue.wasm + clusters: + ai-gateway-service: http://ai-gateway-service:18080 + plugin_config: + service_cluster: ai-gateway-service + require_policy: true + # ... +``` + +#### 方式 B：Higress WasmPlugin + +[`wasmplugin-ai-gateway-queue.yaml`](k8s/ai-gateway/wasmplugin-ai-gateway-queue.yaml) 中 `defaultConfig` **不会**自动写入顶层 `clusters`，生产环境需配合 SgFilter 或扩展 CRD 转换逻辑。 + +私有 OCI 仓库需配置 `imagePullSecret`。 + +### 6.5 网关入口测试 + +SpaceGate 使用 `hostNetwork` 时，在节点上访问： + +```bash +curl -i http://:9993/v1/chat/completions \ + -H 'X-RateLimit-Policy: abandon' \ + -H 'X-Tenant-Id: demo' \ + -d '{"prompt":"hi"}' +``` + +### 6.6 生产替换清单 + +| 开发默认 | 生产建议 | +|----------|----------| +| mock-upstream | 真实 LLM Service | +| `AI_REQUIRE_HTTPS_CALLBACK=false` | `true`，回调 URL 必须 HTTPS | +| HTTP Wasm 分发 | OCI 制品 + `oci://` URL | +| 单副本 Redis | 托管 Redis / Sentinel / Cluster | +| 无对象存储 | 配置 S3/MinIO（大 body offload） | + +--- + +## 7. 制作 OCI 制品 + +SpaceGate 支持从 OCI 仓库拉取 Wasm，URL 形式： + +```text +oci:///: +docker://... # 等价 +image://... # 等价 +oci+http://... # 本地非 TLS registry +``` + +接受的 layer 媒体类型： + +- `application/vnd.module.wasm.content.layer.v1+wasm`（推荐） +- `application/vnd.wasm.content.layer.v1+wasm` +- `application/wasm` + +### 7.1 安装 ORAS + +```bash +brew install oras +# 或从 https://github.com/oras-project/oras/releases 下载 +``` + +### 7.2 编译并计算 digest + +```bash +cd spacegate + +cargo build --release \ + --target wasm32-wasip1 \ + --manifest-path plugins/wasm/Cargo.toml \ + -p spacegate_plugin_ai_gateway_queue + +WASM=plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm +shasum -a 256 "$WASM" +``` + +### 7.3 推送到仓库 + +```bash +# 登录（按仓库类型选择） +oras login ghcr.io -u YOUR_USER +# oras login registry.cn-hangzhou.aliyuncs.com +# oras login your-harbor.example.com + +REGISTRY=ghcr.io/your-org +TAG=v1.0.0 + +oras push "${REGISTRY}/ai-gateway-queue:${TAG}" \ + --artifact-type application/vnd.module.wasm.content.layer.v1+wasm \ + "${WASM}:application/wasm" +``` + +推送成功后配置： + +```yaml +url: oci://ghcr.io/your-org/ai-gateway-queue:v1.0.0 +sha256: sha256:<上一步 shasum 输出> # 可选，建议生产开启 +``` + +在 SgFilter / WasmPlugin 中替换 `url` 即可；私有仓库配合 `imagePullSecret`。 + +### 7.4 本地 Registry 测试 + +```bash +docker run -d -p 5000:5000 --name registry registry:2 + +oras push localhost:5000/ai-gateway-queue:v1 \ + --artifact-type application/vnd.module.wasm.content.layer.v1+wasm \ + "${WASM}:application/wasm" +``` + +SpaceGate 配置（本地/insecure）： + +```text +oci+http://localhost:5000/ai-gateway-queue:v1 +``` + +### 7.5 OCI 注意事项 + +| 项 | 说明 | +|----|------| +| Docker Hub | 通常 **不支持** Wasm OCI Artifact，请用 GHCR / Harbor / ACR / ECR 等 | +| 与容器镜像区别 | OCI Artifact 是单层 Wasm 文件，不是 `docker build` 的应用镜像 | +| ai-gateway-service 镜像 | 仍用 [`build-images.sh`](k8s/ai-gateway/build-images.sh) 单独构建 | +| 版本更新 | 改 tag 重新 push；或在配置中更新 `sha256` / `module_cache_key` 触发重新拉取 | + +### 7.6 一键推送脚本（可选） + +```bash +#!/usr/bin/env bash +set -euo pipefail +REGISTRY="${REGISTRY:?set REGISTRY e.g. ghcr.io/your-org}" +TAG="${TAG:-v1.0.0}" +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +WASM="$ROOT/plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm" + +cd "$ROOT" +cargo build --release --target wasm32-wasip1 \ + --manifest-path plugins/wasm/Cargo.toml \ + -p spacegate_plugin_ai_gateway_queue + +DIGEST=$(shasum -a 256 "$WASM" | awk '{print $1}') + +oras push "${REGISTRY}/ai-gateway-queue:${TAG}" \ + --artifact-type application/vnd.module.wasm.content.layer.v1+wasm \ + "${WASM}:application/wasm" + +echo "url: oci://${REGISTRY}/ai-gateway-queue:${TAG}" +echo "sha256: sha256:${DIGEST}" +``` + +保存为 [`deploy/push-wasm-oci.sh`](push-wasm-oci.sh)（脚本内 `ROOT` 指向 `spacegate` 仓库根目录）后： + +```bash +REGISTRY=ghcr.io/your-org TAG=v1.0.0 ./deploy/push-wasm-oci.sh +``` + +--- + +## 8. 各环境对照表 + +| 环境 | Wasm 分发 | 后端地址配置 | 入口 | +|------|-----------|--------------|------| +| 本地 Cargo | `file://.../plugins/*.wasm` | `127.0.0.1:18080` | `:9993` SpaceGate | +| Docker | volume 挂载 `plugins/` | `http://ai-gateway-service:18080` | `:9993` / `:9080` 管理端 | +| K8s（HTTP） | `http://ai-gateway-wasm/...` | `http://ai-gateway-service:18080` | Gateway `:9993` | +| K8s / 生产（OCI） | `oci://registry/...:tag` | K8s Service DNS | Gateway `:9993` | + +--- + +## 9. 常见问题 + +**Q: 第一次请求就 429？** +A: 检查插件是否在 Gateway 与 Route **重复挂载**；或测试租户 burst 过小。Admin 设置：`PUT /v1/admin/tenant-rate-limits`。 + +**Q: `:9080` 报 `No such file or directory`？** +A: admin-server 读不到 `/etc/spacegate` 配置，恢复 **工作区根目录** `.docker/ai-gateway-demo` 挂载。 + +**Q: SpaceGate 启动报 JSON parse error？** +A: `plugin/` 目录下有 `.wasm` 文件，应移到 `plugins/` 子目录。 + +**Q: macOS 二进制拷进 Linux 容器失败？** +A: 在 Linux 环境 `docker build` 或使用已构建的 `ai-gateway/service:dev` 镜像。 + +**Q: WasmPlugin 无法连 ai-gateway-service？** +A: Higress WasmPlugin 的 `defaultConfig` 不含 `clusters`，请用 **SgFilter** 或改用 OCI + 完整 spec。 + +--- + +## 10. 目录索引 + +```text +spacegate/ +├── plugins/wasm/ai-gateway-queue/ # Wasm 插件源码 +├── binary/ai-gateway-service/ # 队列/限流后端 +├── resource/ai-gateway-demo/ # 文件模式配置模板 +├── deploy/ +│ ├── README.md # 本文档 +│ └── k8s/ai-gateway/ # K8s manifest + apply.sh +└── docs/ + ├── ai-gateway-queue-test-spec.md # 测试用例 + └── ai-gateway-queue-design-gap-fixlist.md +``` diff --git a/deploy/k8s/ai-gateway/ai-gateway-service.yaml b/deploy/k8s/ai-gateway/ai-gateway-service.yaml new file mode 100644 index 00000000..7efa8e98 --- /dev/null +++ b/deploy/k8s/ai-gateway/ai-gateway-service.yaml @@ -0,0 +1,71 @@ +# ai-gateway-service：限流 / 入队 / Worker / 回调 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ai-gateway-service + labels: + app.kubernetes.io/name: ai-gateway-service + app.kubernetes.io/part-of: ai-gateway +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: ai-gateway-service + template: + metadata: + labels: + app.kubernetes.io/name: ai-gateway-service + spec: + containers: + - name: ai-gateway-service + # 构建：见 deploy/k8s/ai-gateway/build-images.sh 或使用本地镜像 ai-gateway/service:dev + image: ai-gateway/service:dev + imagePullPolicy: IfNotPresent + ports: + - containerPort: 18080 + name: http + env: + - name: REDIS_URL + value: redis://ai-gateway-redis:6379/ + - name: AI_UPSTREAM_BASE_URL + value: http://ai-gateway-mock-upstream:9000 + - name: AI_REQUIRE_HTTPS_CALLBACK + value: "false" + - name: AI_GATEWAY_SERVICE_HOST + value: 0.0.0.0 + - name: AI_GATEWAY_SERVICE_PORT + value: "18080" + - name: RUST_LOG + value: info + readinessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 3 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + memory: 512Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: ai-gateway-service + labels: + app.kubernetes.io/name: ai-gateway-service +spec: + selector: + app.kubernetes.io/name: ai-gateway-service + ports: + - name: http + port: 18080 + targetPort: http diff --git a/deploy/k8s/ai-gateway/apply.sh b/deploy/k8s/ai-gateway/apply.sh new file mode 100755 index 00000000..78d04ee7 --- /dev/null +++ b/deploy/k8s/ai-gateway/apply.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# 编译 Wasm + 打包 ConfigMap + 部署 AI Gateway K8s 栈 +set -euo pipefail +DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$DIR/../../.." && pwd)" +WASM_SRC="$ROOT/plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm" +WASM_DST="$DIR/files/spacegate_plugin_ai_gateway_queue.wasm" + +echo "==> 检查 SpaceGate 前置（namespace / GatewayClass / DaemonSet）" +if ! kubectl get namespace spacegate >/dev/null 2>&1; then + echo "ERROR: namespace 'spacegate' 不存在。请先安装 SpaceGate：" >&2 + echo " kubectl apply -f $ROOT/resource/kube-manifests/namespace.yaml" >&2 + echo " kubectl apply -f $ROOT/resource/kube-manifests/gatewayclass.yaml" >&2 + echo " kubectl apply -f $ROOT/resource/kube-manifests/spacegate-gateway.yaml" >&2 + exit 1 +fi + +echo "==> 编译 ai-gateway-queue Wasm" +cd "$ROOT" +rustup target add wasm32-wasip1 2>/dev/null || true +cargo build --release --target wasm32-wasip1 \ + --manifest-path plugins/wasm/Cargo.toml \ + -p spacegate_plugin_ai_gateway_queue + +mkdir -p "$DIR/files" +cp "$WASM_SRC" "$WASM_DST" + +echo "==> 应用 Kustomize" +kubectl apply -k "$DIR" + +echo "==> 等待 Pod Ready" +kubectl wait --for=condition=ready pod \ + -l app.kubernetes.io/part-of=ai-gateway \ + -n spacegate \ + --timeout=180s + +echo "" +echo "部署完成。验证：" +echo " $DIR/verify.sh" +echo "" +echo "网关入口（SpaceGate hostNetwork 监听 9993）：" +echo " curl -i http://:9993/v1/chat/completions \\" +echo " -H 'X-RateLimit-Policy: abandon' -H 'X-Tenant-Id: demo' -d '{\"prompt\":\"hi\"}'" diff --git a/deploy/k8s/ai-gateway/build-images.sh b/deploy/k8s/ai-gateway/build-images.sh new file mode 100755 index 00000000..466e75f8 --- /dev/null +++ b/deploy/k8s/ai-gateway/build-images.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# 构建 ai-gateway-service Linux 镜像（供 K8s 使用） +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +cd "$ROOT" + +IMAGE="${AI_GATEWAY_SERVICE_IMAGE:-ai-gateway/service:dev}" +DOCKERFILE="$(dirname "$0")/docker/Dockerfile.ai-gateway-service" + +echo "Building ai-gateway-service (linux/amd64) ..." +docker build -f "$DOCKERFILE" \ + --build-arg SPACEGATE_ROOT="$ROOT" \ + -t "$IMAGE" \ + "$ROOT" + +echo "Done. Image: $IMAGE" +echo "For k3d/minikube: import locally, e.g." +echo " k3d image import $IMAGE -c " diff --git a/deploy/k8s/ai-gateway/docker/Dockerfile.ai-gateway-service b/deploy/k8s/ai-gateway/docker/Dockerfile.ai-gateway-service new file mode 100644 index 00000000..ec52d09e --- /dev/null +++ b/deploy/k8s/ai-gateway/docker/Dockerfile.ai-gateway-service @@ -0,0 +1,17 @@ +# ai-gateway-service K8s 镜像（多阶段：Rust 编译 + 最小运行层） +ARG RUST_IMAGE=rust:1-bookworm +ARG RUNTIME_IMAGE=debian:bookworm-slim + +FROM ${RUST_IMAGE} AS builder +ARG SPACEGATE_ROOT=/src +WORKDIR /src +COPY . . +RUN cargo build --release -p ai-gateway-service + +FROM ${RUNTIME_IMAGE} +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates tini \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /src/target/release/ai-gateway-service /usr/local/bin/ai-gateway-service +EXPOSE 18080 +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["/usr/local/bin/ai-gateway-service"] diff --git a/deploy/k8s/ai-gateway/files/.gitkeep b/deploy/k8s/ai-gateway/files/.gitkeep new file mode 100644 index 00000000..79d6aec9 --- /dev/null +++ b/deploy/k8s/ai-gateway/files/.gitkeep @@ -0,0 +1,2 @@ +# 占位：apply.sh 会把编译好的 wasm 复制到此目录 +# 勿手动删除本目录 diff --git a/deploy/k8s/ai-gateway/gateway-ai.yaml b/deploy/k8s/ai-gateway/gateway-ai.yaml new file mode 100644 index 00000000..08cd30d9 --- /dev/null +++ b/deploy/k8s/ai-gateway/gateway-ai.yaml @@ -0,0 +1,17 @@ +# AI 流量入口 Gateway（需已安装 GatewayClass: spacegate） +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: ai-gateway + labels: + app.kubernetes.io/name: ai-gateway + app.kubernetes.io/part-of: ai-gateway +spec: + gatewayClassName: spacegate + listeners: + - name: http + port: 9993 + protocol: HTTP + allowedRoutes: + namespaces: + from: Same diff --git a/deploy/k8s/ai-gateway/httproute-ai.yaml b/deploy/k8s/ai-gateway/httproute-ai.yaml new file mode 100644 index 00000000..9fbf1f23 --- /dev/null +++ b/deploy/k8s/ai-gateway/httproute-ai.yaml @@ -0,0 +1,20 @@ +# /v1/* 路由到 mock 上游；Wasm 插件通过 SgFilter 挂载在本 Route 上（仅挂一次，避免双倍限流） +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: ai-api + labels: + app.kubernetes.io/name: ai-api + app.kubernetes.io/part-of: ai-gateway +spec: + parentRefs: + - name: ai-gateway + namespace: spacegate + rules: + - matches: + - path: + type: PathPrefix + value: /v1/ + backendRefs: + - name: ai-gateway-mock-upstream + port: 9000 diff --git a/deploy/k8s/ai-gateway/kustomization.yaml b/deploy/k8s/ai-gateway/kustomization.yaml new file mode 100644 index 00000000..a6624099 --- /dev/null +++ b/deploy/k8s/ai-gateway/kustomization.yaml @@ -0,0 +1,25 @@ +# AI Gateway 队列限流插件 — K8s 一键部署（Kustomize） +# 前置：spacegate 命名空间、GatewayClass、SpaceGate DaemonSet 已安装 +# 用法：./apply.sh +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: spacegate + +resources: + - redis.yaml + - mock-upstream.yaml + - wasm-server.yaml + - ai-gateway-service.yaml + - gateway-ai.yaml + - httproute-ai.yaml + - sgfilter-ai-gateway-queue.yaml + # 可选：Higress WasmPlugin（不含 clusters，生产建议用 SgFilter） + # - wasmplugin-ai-gateway-queue.yaml + +configMapGenerator: + - name: ai-gateway-queue-wasm + files: + - files/spacegate_plugin_ai_gateway_queue.wasm + options: + disableNameSuffixHash: true diff --git a/deploy/k8s/ai-gateway/mock-upstream.yaml b/deploy/k8s/ai-gateway/mock-upstream.yaml new file mode 100644 index 00000000..f974836f --- /dev/null +++ b/deploy/k8s/ai-gateway/mock-upstream.yaml @@ -0,0 +1,49 @@ +# 模拟上游 LLM（生产环境替换为真实模型 Service） +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ai-gateway-mock-upstream + labels: + app.kubernetes.io/name: ai-gateway-mock-upstream + app.kubernetes.io/part-of: ai-gateway +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: ai-gateway-mock-upstream + template: + metadata: + labels: + app.kubernetes.io/name: ai-gateway-mock-upstream + spec: + containers: + - name: echo + image: hashicorp/http-echo:1.0.0 + args: + - -text={"choices":[{"message":{"content":"ok"}}]} + - -listen=:9000 + ports: + - containerPort: 9000 + name: http + readinessProbe: + tcpSocket: + port: http + periodSeconds: 5 + resources: + requests: + cpu: 10m + memory: 16Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: ai-gateway-mock-upstream + labels: + app.kubernetes.io/name: ai-gateway-mock-upstream +spec: + selector: + app.kubernetes.io/name: ai-gateway-mock-upstream + ports: + - name: http + port: 9000 + targetPort: http diff --git a/deploy/k8s/ai-gateway/redis.yaml b/deploy/k8s/ai-gateway/redis.yaml new file mode 100644 index 00000000..d0babe4d --- /dev/null +++ b/deploy/k8s/ai-gateway/redis.yaml @@ -0,0 +1,54 @@ +# Redis 7+（队列 / 限流 / 结果存储） +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ai-gateway-redis + labels: + app.kubernetes.io/name: ai-gateway-redis + app.kubernetes.io/part-of: ai-gateway +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: ai-gateway-redis + template: + metadata: + labels: + app.kubernetes.io/name: ai-gateway-redis + spec: + containers: + - name: redis + image: redis:7-alpine + ports: + - containerPort: 6379 + name: redis + readinessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 3 + periodSeconds: 5 + livenessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 256Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: ai-gateway-redis + labels: + app.kubernetes.io/name: ai-gateway-redis +spec: + selector: + app.kubernetes.io/name: ai-gateway-redis + ports: + - name: redis + port: 6379 + targetPort: redis diff --git a/deploy/k8s/ai-gateway/sgfilter-ai-gateway-queue.yaml b/deploy/k8s/ai-gateway/sgfilter-ai-gateway-queue.yaml new file mode 100644 index 00000000..1d496f54 --- /dev/null +++ b/deploy/k8s/ai-gateway/sgfilter-ai-gateway-queue.yaml @@ -0,0 +1,42 @@ +# ai-gateway-queue Wasm 插件（SgFilter 含完整 clusters 映射，推荐 K8s 用法） +apiVersion: spacegate.idealworld.group/v1 +kind: SgFilter +metadata: + name: ai-gateway-queue + labels: + app.kubernetes.io/name: ai-gateway-queue + app.kubernetes.io/part-of: ai-gateway +spec: + targetRefs: + - kind: httproute + name: ai-api + namespace: spacegate + filters: + - code: wasm + name: ai-gateway-queue + enable: true + config: + url: http://ai-gateway-wasm/spacegate_plugin_ai_gateway_queue.wasm + fail_strategy: fail_close + validate_on_create: false + plugin_name: ai-gateway-queue + plugin_root_id: ai-gateway-queue-root + plugin_vm_id: ai-gateway-queue-vm + vm_pool_size: 4 + wait_vm_pool_size: 4 + limits: + max_memory_pages: 64 + fuel_per_call: 20000000 + epoch_timeout_millis: 50 + max_body_bytes: 33554432 + max_pending_calls: 1 + plugin_config: + service_cluster: ai-gateway-service + service_authority: ai-gateway-service + rate_limit_path: /v1/ratelimit/check + enqueue_path: /v1/queue/enqueue + wait_path: /v1/queue/enqueue-and-wait + service_timeout_ms: 65000 + require_policy: true + clusters: + ai-gateway-service: http://ai-gateway-service:18080 diff --git a/deploy/k8s/ai-gateway/verify.sh b/deploy/k8s/ai-gateway/verify.sh new file mode 100755 index 00000000..b63df86c --- /dev/null +++ b/deploy/k8s/ai-gateway/verify.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# K8s 部署后冒烟验证 +set -euo pipefail +DIR="$(cd "$(dirname "$0")" && pwd)" +NS=spacegate +GW="http://127.0.0.1:9993/v1/chat/completions" + +pass=0 +fail=0 +check() { + local name="$1" expect="$2" got="$3" + if [[ "$got" == "$expect" ]]; then + echo "✅ $name ($got)" + pass=$((pass + 1)) + else + echo "❌ $name 期望=$expect 实际=$got" + fail=$((fail + 1)) + fi +} + +echo "==> 后端 health" +curl -sf "http://127.0.0.1:18080/healthz" >/dev/null 2>&1 \ + && echo "✅ ai-gateway-service /healthz（需 port-forward 或 hostNetwork 可达）" \ + || kubectl exec -n "$NS" deploy/ai-gateway-service -- wget -qO- http://127.0.0.1:18080/healthz >/dev/null \ + && echo "✅ ai-gateway-service /healthz（集群内）" \ + || { echo "⚠️ 跳过直连 health（请 kubectl port-forward svc/ai-gateway-service 18080:18080）"; } + +T="k8s-verify-$RANDOM" +curl -sf -X PUT "http://127.0.0.1:18080/v1/admin/tenant-rate-limits" \ + -H 'Content-Type: application/json' \ + -d "{\"tenant\":\"$T\",\"rps\":5,\"burst\":10}" >/dev/null 2>&1 \ + || kubectl port-forward -n "$NS" svc/ai-gateway-service 18080:18080 >/tmp/pf.log 2>&1 & +PF=$! +sleep 2 +curl -sf -X PUT "http://127.0.0.1:18080/v1/admin/tenant-rate-limits" \ + -H 'Content-Type: application/json' \ + -d "{\"tenant\":\"$T\",\"rps\":5,\"burst\":10}" >/dev/null || true + +echo "==> 网关插件（tenant=$T）" +check "缺 Policy" 400 "$(curl -s -o /dev/null -w '%{http_code}' -X POST "$GW" -H "X-Tenant-Id: $T" -H 'Content-Type: application/json' -d '{}')" +check "abandon 配额内" 200 "$(curl -s -o /dev/null -w '%{http_code}' -X POST "$GW" -H 'X-RateLimit-Policy: abandon' -H "X-Tenant-Id: $T" -H 'Content-Type: application/json' -d '{"p":1}')" + +for i in $(seq 1 8); do + curl -s -o /dev/null -X POST "$GW" -H 'X-RateLimit-Policy: abandon' -H "X-Tenant-Id: $T" -H 'Content-Type: application/json' -d "{\"p\":$i}" || true +done +check "abandon 超额" 429 "$(curl -s -o /dev/null -w '%{http_code}' -X POST "$GW" -H 'X-RateLimit-Policy: abandon' -H "X-Tenant-Id: $T" -H 'Content-Type: application/json' -d '{"p":99}')" + +kill "$PF" 2>/dev/null || true +echo "=== $pass 通过, $fail 失败 ===" +exit "$fail" diff --git a/deploy/k8s/ai-gateway/wasm-server.yaml b/deploy/k8s/ai-gateway/wasm-server.yaml new file mode 100644 index 00000000..9fbd8f5b --- /dev/null +++ b/deploy/k8s/ai-gateway/wasm-server.yaml @@ -0,0 +1,57 @@ +# 集群内 HTTP 分发 Wasm 二进制（SpaceGate 通过 http:// 拉取，无需改 DaemonSet 挂载） +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ai-gateway-wasm + labels: + app.kubernetes.io/name: ai-gateway-wasm + app.kubernetes.io/part-of: ai-gateway +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: ai-gateway-wasm + template: + metadata: + labels: + app.kubernetes.io/name: ai-gateway-wasm + spec: + containers: + - name: nginx + image: nginx:1.27-alpine + ports: + - containerPort: 80 + name: http + volumeMounts: + - name: wasm + mountPath: /usr/share/nginx/html + readOnly: true + readinessProbe: + httpGet: + path: /spacegate_plugin_ai_gateway_queue.wasm + port: http + initialDelaySeconds: 2 + periodSeconds: 5 + resources: + requests: + cpu: 10m + memory: 32Mi + volumes: + - name: wasm + configMap: + name: ai-gateway-queue-wasm + defaultMode: 0444 +--- +apiVersion: v1 +kind: Service +metadata: + name: ai-gateway-wasm + labels: + app.kubernetes.io/name: ai-gateway-wasm +spec: + selector: + app.kubernetes.io/name: ai-gateway-wasm + ports: + - name: http + port: 80 + targetPort: http diff --git a/deploy/k8s/ai-gateway/wasmplugin-ai-gateway-queue.yaml b/deploy/k8s/ai-gateway/wasmplugin-ai-gateway-queue.yaml new file mode 100644 index 00000000..20e10f19 --- /dev/null +++ b/deploy/k8s/ai-gateway/wasmplugin-ai-gateway-queue.yaml @@ -0,0 +1,27 @@ +# 可选：Higress WasmPlugin 方式（defaultConfig 不含 clusters，需改用 SgFilter 或扩展 CRD 转换） +# 默认注释掉，见 kustomization.yaml +apiVersion: extensions.higress.io/v1alpha1 +kind: WasmPlugin +metadata: + name: ai-gateway-queue + labels: + app.kubernetes.io/name: ai-gateway-queue + app.kubernetes.io/part-of: ai-gateway +spec: + url: http://ai-gateway-wasm/spacegate_plugin_ai_gateway_queue.wasm + pluginName: ai-gateway-queue + failStrategy: FAIL_CLOSE + phase: AUTHZ + priority: 100 + defaultConfigDisable: true + matchRules: + - ingress: + - ai-api + config: + service_cluster: ai-gateway-service + service_authority: ai-gateway-service + rate_limit_path: /v1/ratelimit/check + enqueue_path: /v1/queue/enqueue + wait_path: /v1/queue/enqueue-and-wait + service_timeout_ms: 65000 + require_policy: true diff --git a/deploy/push-wasm-oci.sh b/deploy/push-wasm-oci.sh new file mode 100755 index 00000000..e17614b1 --- /dev/null +++ b/deploy/push-wasm-oci.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# 编译 ai-gateway-queue Wasm 并推送到 OCI 仓库（需 oras + 仓库登录） +set -euo pipefail + +REGISTRY="${REGISTRY:?请设置 REGISTRY，例如 ghcr.io/your-org}" +TAG="${TAG:-v1.0.0}" +IMAGE="${IMAGE:-ai-gateway-queue}" +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +WASM="$ROOT/plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm" + +if ! command -v oras >/dev/null 2>&1; then + echo "ERROR: 未找到 oras。安装: brew install oras" >&2 + exit 1 +fi + +echo "==> 编译 Wasm" +cd "$ROOT" +rustup target add wasm32-wasip1 2>/dev/null || true +cargo build --release \ + --target wasm32-wasip1 \ + --manifest-path plugins/wasm/Cargo.toml \ + -p spacegate_plugin_ai_gateway_queue + +DIGEST=$(shasum -a 256 "$WASM" | awk '{print $1}') +REF="${REGISTRY}/${IMAGE}:${TAG}" + +echo "==> 推送到 ${REF}" +oras push "$REF" \ + --artifact-type application/vnd.module.wasm.content.layer.v1+wasm \ + "${WASM}:application/wasm" + +echo "" +echo "推送完成。在 SgFilter / WasmPlugin 中使用：" +echo " url: oci://${REF}" +echo " sha256: sha256:${DIGEST}" diff --git a/docs/ai-gateway-queue-design-gap-fixlist.md b/docs/ai-gateway-queue-design-gap-fixlist.md new file mode 100644 index 00000000..2ac77740 --- /dev/null +++ b/docs/ai-gateway-queue-design-gap-fixlist.md @@ -0,0 +1,644 @@ +# AI Gateway Queue — 设计与代码差距修复清单 + +> 对照文档：`ai-gateway-queue-design.md`（桌面版） +> 审计范围：`spacegate/plugins/wasm/ai-gateway-queue` + `spacegate/binary/ai-gateway-service` +> 生成日期：2026-05-23 + +本文档将设计文档与当前实现的差异整理为**可执行的修复项**，按优先级排序。每项包含：差距说明、建议改法、涉及文件、验收标准、依赖关系。 + +--- + +## 优先级说明 + +| 级别 | 含义 | 建议节奏 | +|------|------|----------| +| **P0** | 核心语义错误，影响限流/队列正确性 | 立即修复 | +| **P1** | 设计明确要求的能力缺失或明显性能/可靠性缺口 | 下一迭代 | +| **P2** | 行为/格式与设计有差异，但不阻断主流程 | 按需排期 | +| **P3** | 文档、默认值、观测增强 | 低优先级 | + +--- + +## P0 — 核心语义 + +### GAP-001：queue/wait 入队前未做令牌桶准入判定 + +**设计期望** + +- 概述：`Gateway → [Rate Limiter] → Redis Stream → Worker → LLM` +- 限流策略表：三种模式在「触发限流时」分别 429 / 202 / 阻塞等待 +- abandon 示例：未触发限流 → 正常 LLM 响应；触发限流 → 429 + +**当前行为** + +- `abandon`：Wasm 调 `/v1/ratelimit/check`，通过则直通上游 ✅ +- `queue` / `wait`：Wasm **直接**调 `/v1/queue/enqueue` 或 `/v1/queue/enqueue-and-wait`，**不做限流判断**，所有请求全量入队 ❌ + +**建议改法** + +1. **方案 A（推荐，改 Wasm 插件）** + - `queue` / `wait` 在入队前先调 `/v1/ratelimit/check`（与 abandon 共用同一接口） + - `allowed: true` → `resume_http_request()` 直通上游（配额内直通） + - `allowed: false` → + - `queue` → 调 enqueue，返回 202 + - `wait` → 调 enqueue-and-wait，阻塞等待 + +2. **方案 B（改 service 入队接口）** + - 在 `enqueue_job()` 开头内联令牌桶逻辑；`allowed: true` 时同步调 upstream 并返回 200（wait）或直接 proxy（需 Gateway 配合，改动面更大） + +**涉及文件** + +- `spacegate/plugins/wasm/ai-gateway-queue/src/lib.rs`（主改） +- `spacegate/plugins/wasm/ai-gateway-queue/README.md` +- `spacegate/binary/ai-gateway-service/src/app/handlers.rs`（若采用方案 B） +- `spacegate/binary/ai-gateway-service/src/app/queue.rs`（若采用方案 B） +- 集成测试 / e2e 脚本 + +**验收标准** + +- [ ] 租户配额内 + `queue` 策略 → **200**，响应来自上游，**不入队** +- [ ] 租户配额内 + `wait` 策略 → **200**，同步上游响应，**不入队** +- [ ] 租户超额 + `queue` → **202** + `X-Job-Id` +- [ ] 租户超额 + `wait` → 入队等待或 **504** 超时 +- [ ] `abandon` 行为保持不变 +- [ ] `rate_limited_total{policy,tenant}` 在 queue/wait 超额入队时也有计数 + +**依赖**：无（应最先做） + +**备注**：需与设计方确认 queue 示例中「无论是否触发限流都 202」是否作废；若保留该语义，则 queue 模式不做 GAP-001 直通，仅 wait/abandon 对齐。 + +--- + +### GAP-002：queue 模式语义与设计文档内部矛盾需定稿 + +**设计矛盾点** + +- 策略对比表：「限流时入队」 +- queue 示例：**「立即返回（无论是否触发限流）」** → 202 + +**当前行为** + +- 与 queue 示例一致：所有 queue 请求都 202 入队 + +**建议改法** + +- **产品/架构定稿二选一**，写入设计文档 v2： + - **模式 Q1（异步优先）**：queue 永远异步入队，不做直通（维持现状） + - **模式 Q2（配额内直通）**：配额内直通，超额才 202（需 GAP-001） + +**涉及文件** + +- 设计文档（外部） +- `spacegate/plugins/wasm/ai-gateway-queue/README.md` +- 前端配置手册 / Admin 文案 + +**验收标准** + +- [ ] 设计文档消除内部矛盾 +- [ ] README、前端说明与定稿一致 +- [ ] 测试用例覆盖定稿语义 + +**依赖**：阻塞 GAP-001 的实现细节 + +--- + +### GAP-003：多租户配额叠加无全局容量保护 + +**设计期望** + +- 设计强调租户隔离，未写全局上限；但生产上多租户「各自配额内」叠加仍可能打满上游 + +**当前行为** + +- 仅 per-tenant 令牌桶；无 cluster 级总 RPS / 总并发 Semaphore +- `abandon` 直通不受 `worker_concurrency` 约束 + +**建议改法** + +1. 增加 **全局令牌桶**（Redis key 如 `ai:global:ratelimit:tokens`），在 `/v1/ratelimit/check` 中 **先扣全局、再扣租户** +2. 增加 **upstream 并发 Semaphore**（`AI_UPSTREAM_MAX_INFLIGHT`），abandon 直通与 Worker 共享 +3. `/metrics` 暴露 `global_rate_limited_total`、`upstream_inflight` + +**涉及文件** + +- `spacegate/binary/ai-gateway-service/src/app/handlers.rs` +- `spacegate/binary/ai-gateway-service/src/app/types.rs`（Lua 或新函数） +- `spacegate/binary/ai-gateway-service/src/app/config.rs` +- `spacegate/binary/ai-gateway-service/config/ai-gateway-service.example.toml` +- `spacegate/binary/ai-gateway-service/src/app/queue.rs`（Worker 侧 acquire permit） + +**验收标准** + +- [ ] 100 个租户各在配额内，全局上限触发后后续请求按策略 429/入队/等待 +- [ ] 指标可观测全局拒绝次数 +- [ ] 配置可独立调整全局 RPS 与 upstream inflight + +**依赖**：建议在 GAP-001 之后 + +--- + +## P1 — 性能与可靠性 + +### GAP-004：S3 multipart 上传与 XADD 顺序执行，非设计所述并发 + +**设计期望** + +> 入队（S3 卸载）：S3 PutObject 与 XADD 并发执行，瓶颈在 S3 + +**当前行为** + +- `store_body()` 完整完成后才 `XADD` + +**建议改法** + +- 小 refactor：`store_body` 返回 `(BodyLocation, future)` 或在超阈值时： + 1. 先 `XADD` 占位 entry（status=uploading）或 + 2. 并行：`tokio::join!(multipart_upload, prepare_metadata)`，最后 XADD +- 最小改动：XADD 只写 ref/metadata，body 上传异步完成后更新 entry 或 Worker 按 ref 拉取（Worker 已支持 ref） + +**涉及文件** + +- `spacegate/binary/ai-gateway-service/src/app/queue.rs` +- `spacegate/binary/ai-gateway-service/src/app/object_store.rs` + +**验收标准** + +- [ ] 大 body 场景 enqueue P99 不因「上传完成 + XADD 串行」线性叠加 +- [ ] 上传失败时 entry 不处于不可消费状态（abort + DLQ 或重试） + +**依赖**：无 + +--- + +### GAP-005：wait 模式每请求新建 SubscriberClient，未实现连接复用 + +**设计期望** + +> 1000 个 wait 并发共享同一物理连接（fred 多路复用订阅） + +**当前行为** + +- 每次 `enqueue_and_wait` 调用 `build_subscriber_client()` 新建连接 + +**建议改法** + +- 在 `AppState` 中维护 **共享 SubscriberClient 池** 或单例 multiplexer +- 按 `result:{job_id}` channel 注册/oneshot 等待，避免 per-request 连接 +- 注意：fred API 下订阅与命令连接分离的要求仍满足 + +**涉及文件** + +- `spacegate/binary/ai-gateway-service/src/app/handlers.rs` +- `spacegate/binary/ai-gateway-service/src/app/runtime.rs`（AppState 初始化） +- `spacegate/binary/ai-gateway-service/src/app/util.rs` +- 新增 `wait_subscriber.rs`（可选） + +**验收标准** + +- [ ] 100 并发 wait 时 Redis 连接数不随请求线性增长 +- [ ] 竞态保险（subscribe 后 get result）仍正确 +- [ ] 超时后 subscriber 无泄漏 + +**依赖**：无 + +--- + +### GAP-006：Worker XREADGROUP 读 5 条但串行处理 + +**设计期望** + +> 每次 XREADGROUP 取 5 条，**批量并发处理** + +**当前行为** + +- `read_worker_stream` 循环内逐条 `process_stream_entry`（串行） + +**建议改法** + +- 对同一 batch 用 `FuturesUnordered` / `tokio::spawn` 并发处理 +- 仍受 `worker_concurrency` 或独立 `worker_inflight` Semaphore 约束 + +**涉及文件** + +- `spacegate/binary/ai-gateway-service/src/app/queue.rs` + +**验收标准** + +- [ ] 队列积压时 Worker 吞吐随 concurrency 提升 +- [ ] job lease 机制下无重复执行 +- [ ] upstream inflight（GAP-003）不被突破 + +**依赖**：建议与 GAP-003 一并设计 + +--- + +### GAP-007：未配置 object_store 时大 body 仍 inline 进 Redis + +**设计期望** + +- 超 128KB 应 offload 到 S3；Redis entry 只存 ref + +**当前行为** + +- 仅当 `object_store.endpoint` 配置存在时才 multipart；否则 >128KB 仍 base64 写入 Stream + +**建议改法** + +- 启动时：若 `inline_threshold` 较小但未配 object_store，**warn 或 fail_fast**（生产配置） +- 或：超阈值且无 S3 时拒绝入队并返回 **413 Payload Too Large** + +**涉及文件** + +- `spacegate/binary/ai-gateway-service/src/app/object_store.rs` +- `spacegate/binary/ai-gateway-service/src/app/config.rs`（校验） +- `spacegate/binary/ai-gateway-service/config/ai-gateway-service.example.toml` + +**验收标准** + +- [ ] 生产配置下 >128KB 请求不会把大 payload 塞进 Redis +- [ ] 本地无 MinIO 时行为明确（拒绝或强制配 endpoint） + +**依赖**：无 + +--- + +## P2 — 协议与行为对齐 + +### GAP-008：`rate_limited_total{policy,tenant}` 仅 abandon 路径计数 + +**设计期望** + +- 监控：各策略触发限流次数 + +**当前行为** + +- 仅在 `check_rate_limit` handler 内 increment；queue/wait 超额入队不计数 + +**建议改法** + +- GAP-001 完成后，在「超额转 queue/wait 分支」同样 `inc_labeled` +- 或抽取 `record_rate_limited(policy, tenant)` 共用 + +**涉及文件** + +- `spacegate/binary/ai-gateway-service/src/app/handlers.rs` +- `spacegate/plugins/wasm/ai-gateway-queue/src/lib.rs`（若 Wasm 侧判定） + +**验收标准** + +- [ ] queue/wait 因配额拒绝而入队时，`rate_limited_total{policy="queue",tenant="..."}` 递增 + +**依赖**：GAP-001 + +--- + +### GAP-009：回调 JSON 与设计示例字段不完全一致 + +**设计期望** + +```json +{ + "job_id": "...", + "status": "completed", + "result": { ...LLM 响应... }, + "completed_at": "2024-01-01T12:00:01Z" +} +``` + +**当前行为** + +- 额外字段：`http_status`、`headers`、`body_base64`、`completed_at_ms`、`error` + +**建议改法** + +- **方案 A**：文档化当前 schema 为正式 API（推荐，向后兼容） +- **方案 B**：增加 `callback_format=v1|v2` 或 Accept 头切换精简格式 + +**涉及文件** + +- `spacegate/binary/ai-gateway-service/src/app/callback.rs` +- `spacegate/binary/ai-gateway-service/README.md` + +**验收标准** + +- [ ] API 文档与实现一致 +- [ ] 若有 v1 精简格式，集成测试覆盖 + +**依赖**：无 + +--- + +### GAP-010：job_id 格式与设计示例不一致 + +**设计期望** + +- 示例：`01J8XYZABC`（类 ULID） + +**当前行为** + +- `{timestamp_hex}{counter_hex}` + +**建议改法** + +- 改用 ULID / UUID v7；或保留现状并更新设计文档 + +**涉及文件** + +- `spacegate/binary/ai-gateway-service/src/app/util.rs`（`new_job_id`） + +**验收标准** + +- [ ] job_id 全局唯一、可排序（若用 ULID） +- [ ] 旧 job 查询不受影响（无需迁移） + +**依赖**：无 + +--- + +### GAP-011：`X-RateLimit-Policy` 可通过配置绕过 + +**设计期望** + +- 请求头表格：Policy **必填** + +**当前行为** + +- `require_policy=false` 且无 default 时 Wasm `Action::Continue` 完全 bypass + +**建议改法** + +- 生产 preset：`require_policy=true` 且文档标注勿关闭 +- 或移除 bypass 路径，仅允许 `default_policy` fallback + +**涉及文件** + +- `spacegate/plugins/wasm/ai-gateway-queue/src/lib.rs` +- Admin 前端默认值 / 校验 + +**验收标准** + +- [ ] 生产配置无法意外 bypass 插件 +- [ ] 缺少 policy 一律 400 + +**依赖**：无 + +--- + +### GAP-012：HTTPS 回调要求可关闭 + +**设计期望** + +- `X-Callback-URL` 需 HTTPS + +**当前行为** + +- `require_https_callback` 默认 true，可 env 关闭 + +**建议改法** + +- 生产 profile 强制 HTTPS；dev profile 允许 HTTP +- 配置校验：非 dev 且 `require_https=false` 启动 warning/error + +**涉及文件** + +- `spacegate/binary/ai-gateway-service/src/app/config.rs` +- `spacegate/binary/ai-gateway-service/src/app/queue.rs`（`validate_callback_url`） + +**验收标准** + +- [ ] 生产启动检查通过 +- [ ] 本地 `AI_REQUIRE_HTTPS_CALLBACK=false` 仍可用 + +**依赖**：无 + +--- + +### GAP-013：令牌桶粒度设计写「仅 Tenant」，实现为 tenant+model+path + +**设计期望** + +- 限流粒度按 `X-Tenant-Id` 隔离 + +**当前行为** + +- Redis key：`ai:ratelimit:{tenant}:{model}:{path}` + Admin 多维规则 + +**建议改法** + +- **推荐**：更新设计文档 v2，声明更细粒度为 intentional enhancement +- 若需严格 tenant-only：增加配置 `rate_limit_granularity=tenant|tenant_model_path` + +**涉及文件** + +- `spacegate/binary/ai-gateway-service/src/app/handlers.rs` +- 设计文档 + +**验收标准** + +- [ ] 文档与实现一致 +- [ ] 可选配置切换粒度（若做） + +**依赖**：无 + +--- + +## P3 — 默认配置、观测与文档 + +### GAP-014：优先级 Stream 默认关闭 + +**设计期望** + +- 扩展：多 Stream 优先级（high/low） + +**当前行为** + +- `enable_priority_streams` 默认 `false` + +**建议改法** + +- 生产 example toml 设为 `true` +- 或 Wasm `plugin_config.priority.enabled` 与 service 配置联动文档化 + +**涉及文件** + +- `spacegate/binary/ai-gateway-service/config/ai-gateway-service.example.toml` +- `spacegate/plugins/wasm/ai-gateway-queue/README.md` + +**验收标准** + +- [ ] 启用后 high/low stream 有深度指标 +- [ ] Worker 按权重消费 + +**依赖**：无 + +--- + +### GAP-015：监控指标命名与设计略有差异 + +**设计期望** + +- `enqueue_latency_ms{policy,size_bucket}` 等 + +**当前行为** + +- Prometheus 文本 + `_bucket{le=...}` histogram 风格；部分为 counter + +**建议改法** + +- 导出与设计对齐的 gauge/histogram（OpenMetrics） +- 或更新设计文档指标名 + +**涉及文件** + +- `spacegate/binary/ai-gateway-service/src/app/handlers.rs`（`/metrics`） +- `spacegate/binary/ai-gateway-service/src/app/metrics.rs` + +**验收标准** + +- [ ] Grafana 面板可按设计指标名查询 +- [ ] `queue_depth > 1000`、`pel_size > 100` 告警规则可配置 + +**依赖**：无 + +--- + +### GAP-016：Redis 版本未校验 + +**设计期望** + +- Redis 7+（Stream、Pub/Sub） + +**当前行为** + +- 运行时未检查版本 + +**建议改法** + +- 启动时 `INFO server` 检查 major >= 7，否则 warn/error + +**涉及文件** + +- `spacegate/binary/ai-gateway-service/src/app/runtime.rs` + +**验收标准** + +- [ ] Redis 6 启动给出明确错误信息 + +**依赖**：无 + +--- + +### GAP-017：Wasm 层 README / 插件文档与实现对齐 + +**当前缺口** + +- README 仍描述「超额入队」，未说明 queue/wait 当前全量入队 +- 未说明 abandon 与 queue/wait 限流路径差异 + +**建议改法** + +- GAP-001 / GAP-002 定稿后一次性更新： + - `spacegate/plugins/wasm/ai-gateway-queue/README.md` + - Admin 内嵌 readme API 同源 + - `spacegate/docs/` 前端配置手册（若有） + +**验收标准** + +- [ ] 文档描述与代码行为一致 +- [ ] curl 示例可 copy 运行通过 + +**依赖**：GAP-001、GAP-002 + +--- + +## 建议实施顺序（Roadmap） + +```text +Phase 0 — 定稿（1-2 天） + GAP-002 queue 模式语义定稿 + GAP-013 限流粒度文档对齐 + +Phase 1 — 核心正确性（1-2 周） + GAP-001 queue/wait 入队前令牌桶（或确认 Q1 不做） + GAP-008 限流指标补全 + GAP-003 全局容量保护 + GAP-011 生产禁止 bypass policy + +Phase 2 — 性能与可靠性（1-2 周） + GAP-005 wait Subscriber 连接复用 + GAP-006 Worker 批量并发 + GAP-004 S3 + XADD 并发 + GAP-007 大 body 无 S3 保护 + +Phase 3 — 对齐与 polish（按需） + GAP-009 ~ GAP-017 +``` + +--- + +## 测试清单（每项修复必跑） + +| 场景 | 命令/用例 | +|------|-----------| +| abandon 配额内 | Policy=abandon，RPS 内 → 200 来自 upstream | +| abandon 超额 | → 429 + Retry-After | +| queue 配额内 | 定稿 Q2：200 直通；Q1：202 | +| queue 超额 | → 202 + callback | +| wait 配额内 | 定稿 Q2：200 同步；否则入队等待 | +| wait 超额/超时 | → 504 + poll_url，job 仍完成 | +| 大 body offload | >128KB + MinIO → Redis 仅 ref | +| Worker 崩溃 | kill worker → XAUTOCLAIM 重认领 | +| 回调失败 | 不可达 URL → retry stream → DLQ | +| 多租户叠加 | 触发 GAP-003 全局上限 | + +现有脚本参考： + +- `spacegate/binary/ai-gateway-service` 下 unit tests +- `tests/queue-object-store-e2e.sh`（若有） + +--- + +## 变更影响矩阵 + +| GAP | Wasm 插件 | ai-gateway-service | Admin 前端 | 破坏性 | +|-----|-----------|-------------------|------------|--------| +| 001 | ✅ | 可选 | 文案 | **高**（queue 从全 202 变为配额内 200） | +| 002 | — | — | 文案 | 产品决策 | +| 003 | — | ✅ | 可选配额 UI | 中 | +| 004 | — | ✅ | — | 低 | +| 005 | — | ✅ | — | 低 | +| 006 | — | ✅ | — | 低 | +| 007 | — | ✅ | — | 中（大 body 可能从能入队变 413） | +| 008 | 可选 | ✅ | — | 低 | +| 011 | ✅ | — | ✅ | 中 | +| 014 | 配置 | ✅ | ✅ | 低 | + +--- + +## 开放问题（实施前需确认） + +1. **queue 模式**：永远 202（Q1）还是配额内直通（Q2）？ +2. **wait 模式**：配额内是否应同步直通上游，还是始终走队列（便于统一 observability）？ +3. **全局容量**：是否需要独立配置项暴露给 Admin，还是仅 ops 环境变量？ +4. **GAP-001 方案 A vs B**：限流判定放在 Wasm 还是 service 入队接口内？ +5. **job_id 是否改为 ULID**：有无外部系统已依赖当前 hex 格式？ + +--- + +## 修订记录 + +| 日期 | 说明 | +|------|------| +| 2026-05-23 | 初版：基于设计文档 vs 代码审计生成 | +| 2026-05-24 | **DOC-01/02 定稿**：遵循概述 `Gateway → [Rate Limiter] → …` 与策略表「限流时」语义。三种策略均先过令牌桶；**配额内直通上游**（`resume_http_request`）；超额时 abandon→429、queue→202 入队、wait→入队并阻塞等待。queue 示例「无论是否限流都 202」以策略表为准作废。 | +| 2026-05-24 | **全量差距项实施完成**：G-01~G-20、A/Q/W 分项及 DOC 定稿均已落地；`cargo test -p ai-gateway-service` 14/14 通过。详见各模块 commit 与 README 更新。 | + +--- + +## DOC-01 / DOC-02 定稿结论（2026-05-24） + +| 策略 | 配额内（allowed） | 超额（rate limited） | +|------|------------------|---------------------| +| abandon | 直通上游 | 429 | +| queue | 直通上游 | 202 异步入队 | +| wait | 直通上游 | 入队 + Pub/Sub 阻塞等待 | diff --git a/docs/ai-gateway-queue-test-spec.md b/docs/ai-gateway-queue-test-spec.md new file mode 100644 index 00000000..1d41d1ad --- /dev/null +++ b/docs/ai-gateway-queue-test-spec.md @@ -0,0 +1,370 @@ +# AI Gateway 队列 — 测试用例规格 + +**设计文档：** [`/Users/sh.zhang/Documents/ai-gateway-queue-design.md`](/Users/sh.zhang/Documents/ai-gateway-queue-design.md) +**语义基准：** DOC-01/02 定稿（配额内三种策略均直通上游；超额时 abandon→429 / queue→202 / wait→阻塞或 504） + +--- + +## Traceability 矩阵 + +| 设计文档章节 | 用例 ID | 自动化 | +|-------------|---------|--------| +| §限流策略 / 请求头 | TC-HDR-* | Rust IT / Hurl / GW E2E | +| §abandon 示例 | TC-AB-* | Rust IT / Hurl / GW E2E | +| §queue 示例 / 时序 | TC-Q-* | Rust IT / Hurl | +| §wait 示例 / 时序 | TC-W-* | Rust IT / Hurl | +| §核心组件 §1 限流器 | TC-RL-* | Rust IT / Hurl | +| §核心组件 §2 Body | TC-BODY-* | Rust IT / MinIO E2E | +| §核心组件 §3 Stream | TC-Q-* / TC-WK-* | Rust IT | +| §核心组件 §4 Pub/Sub | TC-W-* | Rust IT | +| §性能设计 | TC-BODY-05/07, TC-W-06 | Rust IT | +| §可靠性 | TC-WK-* | Rust IT | +| §监控指标 | TC-MET-* | Rust IT / Hurl | +| §部署 | TC-DEP-* | Shell | +| Wasm 网关层 | TC-GW-* | GW E2E（可选） | + +**图例：** Rust IT = `cargo test --test integration`；Hurl = `tests/hurl/*.hurl`；GW E2E = `scripts/run-gateway-e2e.sh` + +--- + +## 1. 请求头与策略（TC-HDR） + +### TC-HDR-01 缺 Policy 且无 default + +- **设计映射：** §限流策略 — `X-RateLimit-Policy` 必填 +- **前置：** Wasm `default_policy=null`；Service 直接调用入队接口 +- **步骤：** POST `/v1/queue/enqueue`，不带 `x-ratelimit-policy` +- **期望：** 400；Service 侧 bad request（若直接打 service 则 policy 可选但 Wasm 层 400） + +### TC-HDR-02 Policy 非法值 + +- **步骤：** `x-ratelimit-policy: invalid` +- **期望：** Wasm 400 `missing_or_invalid_rate_limit_policy` + +### TC-HDR-03 缺 X-Tenant-Id + +- **步骤：** 任意策略，不带 tenant +- **期望：** Wasm 400 `missing_x_tenant_id`；Service `/v1/ratelimit/check` 400 + +### TC-HDR-04 queue 缺 X-Callback-URL + +- **步骤：** POST `/v1/queue/enqueue`，policy=queue，无 callback +- **期望：** 400 `missing required header x-callback-url` + +### TC-HDR-05 queue 回调非 HTTPS（生产配置） + +- **前置：** `require_https_callback=true` +- **步骤：** `x-callback-url: http://example.com/cb` +- **期望：** 400 `x-callback-url must use https` + +### TC-HDR-06 wait 默认 timeout 60s + +- **前置：** 上游/mock 延迟 >60s；`wait_timeout_secs=60` +- **步骤：** wait 入队并等待 +- **期望：** 504；JSON 含 `error=timeout`、`waited_ms`≈60000 + +### TC-HDR-07 wait 自定义 X-Request-Timeout + +- **步骤：** `x-request-timeout: 2`（测试配置缩短） +- **期望：** ~2s 后 504 + +--- + +## 2. 限流器（TC-RL） + +### TC-RL-01 租户隔离 + +- **设计映射：** §核心组件 §1 — 限流粒度按 X-Tenant-Id +- **前置：** RPS=1, burst=1 +- **步骤：** tenant-A 连续 2 次 check；tenant-B 1 次 check +- **期望：** A 第二次 `allowed=false`；B 第一次 `allowed=true` + +### TC-RL-02 配额内 allowed + +- **步骤：** 首次 check +- **期望：** `{ "allowed": true, "retry_after_ms": 0 }` + +### TC-RL-03 超额与指标 + +- **步骤：** 耗尽 burst 后再 check +- **期望：** `allowed=false`，`retry_after_ms>0`；`/metrics` 中 `rate_limited_total{policy,tenant}` +1 + +### TC-RL-04 burst 超发后拒绝 + +- **前置：** burst=2 +- **步骤：** 连续 3 次 check（同 tenant） +- **期望：** 前 2 次 allowed，第 3 次 denied + +### TC-RL-05 Admin 租户规则覆盖 + +- **步骤：** PUT `/v1/admin/tenant-rate-limits` 设置 tenant 低 RPS；再 check +- **期望：** 新 RPS 生效（更快触发 denied） + +### TC-RL-06 规则 lookup 优先级 + +- **步骤：** 写入 tenant 全局规则 + tenant+model 更严格规则；带 model header check +- **期望：** 使用更具体规则 + +### TC-RL-07 Redis key tenant-only + +- **步骤：** check 后 Redis KEYS `ai:ratelimit:*` +- **期望：** 仅 `ai:ratelimit:{tenant}:tokens` 与 `:ts`；不含 model/path + +--- + +## 3. abandon（TC-AB） + +### TC-AB-01 配额内直通 + +- **设计映射：** §abandon — 未触发限流时正常返回 LLM 响应 +- **步骤：** Wasm policy=abandon，配额内 +- **期望：** 200，body 来自 upstream（非 202/429） + +### TC-AB-02 超额 429 + +- **步骤：** 触发限流 +- **期望：** 429；`Retry-After`；`{"error":"rate_limited","retry_after_ms":N}` + +### TC-AB-03 不调用 enqueue + +- **步骤：** 配额内 abandon；监控 service 日志/无 enqueue 指标增长 +- **期望：** 无 `/v1/queue/enqueue` 调用 + +--- + +## 4. queue（TC-Q） + +### TC-Q-01 配额内直通（定稿） + +- **步骤：** Wasm policy=queue，配额内 +- **期望：** 200 上游响应（**非**设计文档 queue 示例「永远 202」） + +### TC-Q-02 超额 202 入队 + +- **步骤：** 超额 queue +- **期望：** 202；Header `X-Job-Id`；JSON `poll_url=/jobs/{id}/status` + +### TC-Q-03 202 JSON 字段 + +- **期望：** `job_id` 为 ULID 格式；`status=queued`；`poll_url` 正确 + +### TC-Q-04 Worker 回调 + +- **前置：** mock callback server +- **步骤：** 超额入队 → worker 完成 +- **期望：** POST 回调；Header `X-Gateway-Job-Id` + +### TC-Q-05 回调 JSON 四字段 + +- **期望：** `{ job_id, status, result, completed_at }` 仅此四字段（result 为 LLM JSON） + +### TC-Q-06 Stream entry 字段 + +- **步骤：** XREAD 或 XRANGE 读 stream +- **期望：** job_id, body/ref, size, policy, callback_url, headers, created_at 等齐全 + +### TC-Q-07 dev HTTP 回调 + +- **前置：** `require_https_callback=false` +- **步骤：** `http://` callback URL 入队 +- **期望：** 202 + +--- + +## 5. wait（TC-W） + +### TC-W-01 配额内直通 + +- **期望：** 200 上游响应，无入队等待 + +### TC-W-02 超额成功 + +- **步骤：** 超额 wait，worker 正常 +- **期望：** 200；`X-Job-Id`；`X-Queue-Wait-Ms`；LLM body + +### TC-W-03 竞态保险 + +- **前置：** worker 即时完成（0 延迟 upstream） +- **步骤：** enqueue-and-wait +- **期望：** 200（subscribe 前 result 已写入） + +### TC-W-04 超时 504 + +- **前置：** upstream 延迟 > timeout +- **期望：** 504；`error/timeout/job_id/waited_ms/message` + +### TC-W-05 504 后 poll + +- **步骤：** 504 后等待 worker 完成；GET `/jobs/{id}/status` +- **期望：** 200 原始 LLM 响应体 + +### TC-W-06 Pub/Sub 连接复用 smoke + +- **步骤：** 并发 N 个 wait（N=10 smoke） +- **期望：** 全部完成；Redis 连接数无 N 倍 subscriber 连接 + +--- + +## 6. Body 处理（TC-BODY） + +### TC-BODY-01 inline ≤128KB + +- **期望：** storage=inline；Redis entry 含 base64 body + +### TC-BODY-02 S3 卸载 >128KB + +- **前置：** 配置 object_store_endpoint +- **期望：** storage=object；entry 仅 ref；`object_offload_total`+1 + +### TC-BODY-03 无 S3 大 body + +- **期望：** 413 Payload Too Large + +### TC-BODY-04 超 MAX_BODY_BYTES + +- **期望：** 413 + +### TC-BODY-05 S3 与 XADD 并发 + +- **期望：** 入队在合理时间内完成（相对串行基线） + +### TC-BODY-06 multipart 失败 Abort + +- **前置：** mock S3 返回 500 +- **期望：** 入队失败；无成功 XADD + +### TC-BODY-07 body Semaphore + +- **前置：** body_read_concurrency=2（测试配置） +- **步骤：** 3 个并发大 body 入队 +- **期望：** 第三个延迟开始（可选 smoke） + +--- + +## 7. Worker / 可靠性（TC-WK） + +### TC-WK-01 批量并发消费 + +- **步骤：** 一次 XADD 5 条；观察 worker 处理 +- **期望：** 5 条均完成（并发处理） + +### TC-WK-02 XAUTOCLAIM + +- **前置：** reclaim_interval_secs=2（测试）；模拟 PEL 未 ACK +- **期望：** 重认领后重新处理 + +### TC-WK-03 回调失败 → retry stream + +- **前置：** callback URL 不可达 +- **期望：** callback_retry_stream 有 entry + +### TC-WK-04 回调 DLQ + +- **前置：** 超过 max retry +- **期望：** callback_dlq_stream 有 entry + +### TC-WK-05 job DLQ + +- **前置：** max_delivery_attempts=1；反复失败 +- **期望：** job_dlq_stream + +### TC-WK-06 result TTL 120s + +- **前置：** result_ttl_secs=2（测试） +- **步骤：** 完成后等待 TTL；poll +- **期望：** 404 not_found + +### TC-WK-07 优先级 Stream + +- **前置：** enable_priority_streams=true；high/normal 均有积压 +- **期望：** high 优先被消费完 + +--- + +## 8. 监控与部署（TC-MET / TC-DEP） + +### TC-MET-01 metrics 基础 + +- **步骤：** GET `/metrics` +- **期望：** 200；含 `queue_depth`、`pel_size` + +### TC-MET-02 rate_limited 标签 + +- **期望：** `rate_limited_total{policy="...",tenant="..."}` 行存在 + +### TC-MET-03 enqueue_latency 分桶 + +- **期望：** `enqueue_latency_ms_bucket{policy,size_bucket,le=...}` 存在 + +### TC-DEP-01 Redis 6 拒绝 + +- **步骤：** 对 Redis 6 启动 service +- **期望：** 启动失败，明确错误信息 + +### TC-DEP-02 Redis 7+ 通过 + +- **期望：** 正常启动 + +--- + +## 9. Wasm 网关层（TC-GW，可选） + +### TC-GW-01 abandon 超额 + +- **期望：** 429 + +### TC-GW-02 queue 超额 + +- **期望：** 202 + +### TC-GW-03 wait 超额 + +- **期望：** 200 或 504（视 upstream 延迟） + +### TC-GW-04 service 不可达 + +- **期望：** 502 + +--- + +## 运行命令 + +```bash +# 单元测试（无需 Redis） +cd spacegate && cargo test -p ai-gateway-service + +# 集成测试（需 Redis 7+） +./spacegate/binary/ai-gateway-service/scripts/run-integration-tests.sh + +# Hurl 黑盒 +./spacegate/binary/ai-gateway-service/scripts/run-hurl-tests.sh + +# MinIO E2E +./spacegate/binary/ai-gateway-service/scripts/queue-object-store-e2e.sh + +# Wasm 策略逻辑（host） +cd spacegate/plugins/wasm/ai-gateway-queue && cargo test --lib +``` + +--- + +## 修订记录 + +| 日期 | 说明 | +|------|------| +| 2026-05-24 | 初版：55 条 TC-* 用例 + traceability | +| 2026-05-24 | 落地 Rust 集成测试 22 项、Hurl 5 文件、脚本 4 个、Wasm policy host 测试 3 项 | + +## 已实现自动化映射 + +| 用例 ID | Rust IT | Hurl | 脚本 | +|---------|---------|------|------| +| TC-HDR-03~05 | body_store / enqueue_queue | queue | | +| TC-RL-01~07 | ratelimit / admin | ratelimit / admin | | +| TC-Q-02~07 | enqueue_queue | queue | | +| TC-W-02~05 | enqueue_wait | wait | | +| TC-BODY-01/03 | body_store | | | +| TC-WK-01/03 | worker_reliability | | | +| TC-MET-01/02, TC-DEP-02 | metrics | metrics | | +| TC-BODY-02 | | | queue-object-store-e2e.sh | +| TC-GW / TC-HDR-02 | policy host tests | | run-gateway-e2e.sh (stub) | diff --git a/plugins/wasm/ai-gateway-queue/Cargo.toml b/plugins/wasm/ai-gateway-queue/Cargo.toml index b470a411..711a2531 100644 --- a/plugins/wasm/ai-gateway-queue/Cargo.toml +++ b/plugins/wasm/ai-gateway-queue/Cargo.toml @@ -6,7 +6,7 @@ publish.workspace = true description = "AI gateway rate-limit and queue Proxy-Wasm plugin for SpaceGate." [lib] -crate-type = ["cdylib"] +crate-type = ["cdylib", "rlib"] [dependencies] proxy-wasm.workspace = true diff --git a/plugins/wasm/ai-gateway-queue/README.md b/plugins/wasm/ai-gateway-queue/README.md index c6f8767a..eb2cbd52 100644 --- a/plugins/wasm/ai-gateway-queue/README.md +++ b/plugins/wasm/ai-gateway-queue/README.md @@ -1,12 +1,12 @@ # ai-gateway-queue -`ai-gateway-queue` 是一个运行在 SpaceGate Wasm 里的 **AI 请求队列网关**插件：在入口处对 AI 请求按租户做准入判定（基于令牌桶速率），命中后按选定的队列模式把请求分流到 Redis 多优先级队列异步消化，配合回调重试和对象存储 offload 实现无损交付。 +`ai-gateway-queue` 是一个运行在 SpaceGate Wasm 里的 **AI 请求队列网关**插件：在入口处对 AI 请求按租户做令牌桶限流，再根据 `X-RateLimit-Policy` 分流。**三种策略均先调用 `/v1/ratelimit/check`**；配额内直通上游，超额时分别返回 429 / 202 入队 / 入队并阻塞等待。 支持三种队列模式（通过 `X-RateLimit-Policy` 请求头选择，名字保留兼容历史）： -- `abandon`：超额请求直接返回 429（不入队，等价于纯节流闸门） -- `queue`：超额请求入队后立即返回 `202`，结果通过回调或轮询拿到 -- `wait`：超额请求入队后同步等待结果返回（类长轮询） +- `abandon`：配额内直通上游；超额返回 429（不入队） +- `queue`：配额内直通上游；超额入队后立即返回 `202`，结果通过回调或轮询拿到 +- `wait`：配额内直通上游；超额入队后同步等待结果（类长轮询），超时返回 `504` 插件本身不直接访问 Redis，而是通过 `dispatch_http_call` 调用外部队列后端（`ai-gateway-service`），再由该后端处理 Redis Streams、worker 消费、回调重试、结果回收等队列基础设施。 @@ -183,7 +183,7 @@ curl -i http://localhost:9080/your/api \ ### 2. `queue` -请求体进入队列，插件立即返回 `202 Accepted`，响应里会带 `X-Job-Id`。 +先调用限流接口。配额内继续转发到上游（与 `abandon` 相同）；超额时请求体进入队列，插件返回 `202 Accepted`，响应里会带 `X-Job-Id`。 示例： @@ -197,7 +197,7 @@ curl -i http://localhost:9080/your/api \ ### 3. `wait` -请求体进入队列后等待结果返回。成功时直接返回上游响应，超时则返回 `504`。 +先调用限流接口。配额内继续转发到上游；超额时请求体进入队列后等待结果返回。成功时直接返回上游响应，超时则返回 `504`。 示例： @@ -211,10 +211,10 @@ curl -i http://localhost:9080/your/api \ ## 返回行为 -- `400`：缺少必要请求头或策略非法 -- `429`：限流拒绝 -- `202`：队列已接收 -- `200`/`4xx`/`5xx`：`wait` 模式下，由外部服务返回 +- `400`：缺少必要请求头或策略非法（无 `X-RateLimit-Policy` 且无 `default_policy` 时也会 400） +- `429`：`abandon` 超额限流 +- `202`：`queue` 超额入队已接收 +- `200`/`4xx`/`5xx`：配额内三种策略均由上游返回；`wait` 超额完成后也由外部服务返回上游响应 - `502`：外部服务不可达或调用失败 `wait` 成功响应会带 `X-Job-Id` 和 `X-Queue-Wait-Ms`；`queue` 响应会带 `X-Job-Id` 和 `Location`。 @@ -226,9 +226,8 @@ curl -i http://localhost:9080/your/api \ - 优先级队列支持 header、模型、租户规则推导，并由 worker 按权重消费高/普通/低优先级 Stream - Worker 崩溃后通过 `XAUTOCLAIM` 重认领 pending job，并通过 Redis 处理租约避免长任务被重复执行 - 回调失败会进入 `AI_CALLBACK_RETRY_STREAM`，按指数退避重试，超过最大次数后进入 `AI_CALLBACK_DLQ_STREAM` -- 大 body 可通过 `AI_OBJECT_STORE_ENDPOINT` 走 S3-compatible multipart 卸载，Redis Stream 中只保留 `ref` -- `ai-gateway-service` 会流式读取请求体；当前 Wasm 插件转发到外部服务时仍受 `dispatch_http_call` 限制，会在插件侧拿到完整 body 后再发出调用 -- 可通过 Redis key 覆盖租户限流：`ai:tenant:ratelimit:{tenant}:rps` 和 `ai:tenant:ratelimit:{tenant}:burst` +- 大 body 可通过 `AI_OBJECT_STORE_ENDPOINT` 走 S3-compatible multipart 卸载，Redis Stream 中只保留 `ref`；未配置 S3 且 body 超过 `AI_INLINE_THRESHOLD` 时返回 `413` +- 租户限流令牌桶按 `X-Tenant-Id` 隔离；可通过 Admin API `PUT /v1/admin/tenant-rate-limits` 或 Redis 配置键覆盖每租户 RPS/Burst（支持 model/path/policy 维度 lookup，桶 key 仍为 tenant-only） - `/metrics` 暴露 Prometheus 文本指标，包含队列深度、PEL、DLQ、入队延迟、body 大小、wait 超时、回调重试和 worker 处理耗时 ## 调试建议 diff --git a/plugins/wasm/ai-gateway-queue/src/lib.rs b/plugins/wasm/ai-gateway-queue/src/lib.rs index 387e75e7..eeb68519 100644 --- a/plugins/wasm/ai-gateway-queue/src/lib.rs +++ b/plugins/wasm/ai-gateway-queue/src/lib.rs @@ -5,6 +5,9 @@ use proxy_wasm::traits::*; use proxy_wasm::types::*; use serde_json::Value; +mod policy; +use policy::{contains_allowed_true, extract_json_number, normalize_policy, normalize_priority}; + proxy_wasm::main! {{ proxy_wasm::set_log_level(LogLevel::Info); proxy_wasm::set_root_context(|_| -> Box { Box::new(AiGatewayRoot::default()) }); @@ -171,7 +174,8 @@ impl RootContext for AiGatewayRoot { Some(Box::new(AiGatewayHttp { cfg: self.cfg.clone(), pending: None, - deferred_policy: None, + rate_limited_enqueue: None, + body_pending: false, })) } @@ -189,7 +193,7 @@ enum Policy { #[derive(Clone, Copy, PartialEq, Eq)] enum Pending { - RateLimit, + RateLimit { policy: Policy }, Queue, Wait, } @@ -197,7 +201,10 @@ enum Pending { struct AiGatewayHttp { cfg: AiGatewayConfig, pending: Option<(u32, Pending)>, - deferred_policy: Option, + /// 限流拒绝后等待 request body 再入队（queue / wait）。 + rate_limited_enqueue: Option, + /// 请求体尚未读完，需在 body 回调中继续处理。 + body_pending: bool, } impl Context for AiGatewayHttp { @@ -211,7 +218,7 @@ impl Context for AiGatewayHttp { self.pending = None; match pending { - Pending::RateLimit => self.handle_rate_limit_response(body_size), + Pending::RateLimit { policy } => self.handle_rate_limit_response(body_size, policy), Pending::Queue | Pending::Wait => self.forward_service_response(body_size), } } @@ -220,11 +227,9 @@ impl Context for AiGatewayHttp { impl HttpContext for AiGatewayHttp { fn on_http_request_headers(&mut self, _: usize, end_of_stream: bool) -> Action { let Some(policy) = self.request_policy() else { - if self.cfg.require_policy { - self.send_json(400, r#"{"error":"missing_or_invalid_rate_limit_policy"}"#); - return Action::Pause; - } - return Action::Continue; + // 设计文档要求 Policy 必填；仅允许 default_policy 兜底，禁止无策略 bypass。 + self.send_json(400, r#"{"error":"missing_or_invalid_rate_limit_policy"}"#); + return Action::Pause; }; if self.tenant_id().is_none() { @@ -232,52 +237,30 @@ impl HttpContext for AiGatewayHttp { return Action::Pause; } - match policy { - Policy::Abandon => { - if self.dispatch_service_call(Pending::RateLimit, &self.cfg.rate_limit_path.clone(), None) { - Action::Pause - } else { - self.send_json(502, r#"{"error":"rate_limit_service_unavailable"}"#); - Action::Pause - } - } - Policy::Queue | Policy::Wait => { - if end_of_stream { - let pending = if policy == Policy::Queue { Pending::Queue } else { Pending::Wait }; - let path = if policy == Policy::Queue { - self.cfg.enqueue_path.clone() - } else { - self.cfg.wait_path.clone() - }; - if !self.dispatch_service_call(pending, &path, Some(&[])) { - self.send_json(502, r#"{"error":"queue_service_unavailable"}"#); - } - } else { - self.deferred_policy = Some(policy); - } - Action::Pause - } + // 三种策略统一先走令牌桶（DOC-01/02 定稿）。 + self.body_pending = !end_of_stream && matches!(policy, Policy::Queue | Policy::Wait); + if self.dispatch_service_call(Pending::RateLimit { policy }, &self.cfg.rate_limit_path.clone(), None) { + Action::Pause + } else { + self.send_json(502, r#"{"error":"rate_limit_service_unavailable"}"#); + Action::Pause } } fn on_http_request_body(&mut self, body_size: usize, end_of_stream: bool) -> Action { - let Some(policy) = self.deferred_policy else { + if !self.body_pending && self.rate_limited_enqueue.is_none() { return Action::Continue; - }; + } if !end_of_stream { return Action::Pause; } - self.deferred_policy = None; - let body = self.get_http_request_body(0, body_size).unwrap_or_default(); - let pending = if policy == Policy::Queue { Pending::Queue } else { Pending::Wait }; - let path = if policy == Policy::Queue { - self.cfg.enqueue_path.clone() - } else { - self.cfg.wait_path.clone() + self.body_pending = false; + + let Some(policy) = self.rate_limited_enqueue.take() else { + return Action::Continue; }; - if !self.dispatch_service_call(pending, &path, Some(&body)) { - self.send_json(502, r#"{"error":"queue_service_unavailable"}"#); - } + let body = self.get_http_request_body(0, body_size).unwrap_or_default(); + self.dispatch_enqueue(policy, &body); Action::Pause } } @@ -308,6 +291,18 @@ impl AiGatewayHttp { } } + fn dispatch_enqueue(&mut self, policy: Policy, body: &[u8]) { + let pending = if policy == Policy::Queue { Pending::Queue } else { Pending::Wait }; + let path = if policy == Policy::Queue { + self.cfg.enqueue_path.clone() + } else { + self.cfg.wait_path.clone() + }; + if !self.dispatch_service_call(pending, &path, Some(body)) { + self.send_json(502, r#"{"error":"queue_service_unavailable"}"#); + } + } + fn service_headers(&self, path: &str) -> Vec<(String, String)> { let policy = self.request_policy().map(policy_name).unwrap_or("abandon").to_string(); let tenant_id = self.tenant_id().unwrap_or_default(); @@ -362,28 +357,42 @@ impl AiGatewayHttp { normalize_priority(&self.cfg.default_priority) } - fn handle_rate_limit_response(&mut self, body_size: usize) { + fn handle_rate_limit_response(&mut self, body_size: usize, policy: Policy) { let status = self.service_status(); let body = self.get_http_call_response_body(0, body_size).unwrap_or_default(); let text = String::from_utf8_lossy(&body); if status == 200 && contains_allowed_true(&text) { + // 配额内：三种策略均直通上游。 self.resume_http_request(); return; } if status == 200 { - let retry_after_ms = extract_json_number(&text, "retry_after_ms").unwrap_or(1000); - let retry_after_secs = ((retry_after_ms + 999) / 1000).max(1).to_string(); - let body = format!(r#"{{"error":"rate_limited","retry_after_ms":{retry_after_ms}}}"#); - let headers = [ - ("content-type".to_string(), "application/json".to_string()), - ("retry-after".to_string(), retry_after_secs), - ("x-ratelimit-retry-after-ms".to_string(), retry_after_ms.to_string()), - ]; - let headers = headers.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect::>(); - self.send_http_response(429, headers, Some(body.as_bytes())); - } else { - self.send_json(502, r#"{"error":"rate_limit_service_error"}"#); + match policy { + Policy::Abandon => self.send_rate_limited_response(&text), + Policy::Queue | Policy::Wait => { + if self.body_pending { + self.rate_limited_enqueue = Some(policy); + } else { + self.dispatch_enqueue(policy, &[]); + } + } + } + return; } + self.send_json(502, r#"{"error":"rate_limit_service_error"}"#); + } + + fn send_rate_limited_response(&self, text: &str) { + let retry_after_ms = extract_json_number(text, "retry_after_ms").unwrap_or(1000); + let retry_after_secs = ((retry_after_ms + 999) / 1000).max(1).to_string(); + let response_body = format!(r#"{{"error":"rate_limited","retry_after_ms":{retry_after_ms}}}"#); + let headers = [ + ("content-type".to_string(), "application/json".to_string()), + ("retry-after".to_string(), retry_after_secs), + ("x-ratelimit-retry-after-ms".to_string(), retry_after_ms.to_string()), + ]; + let headers = headers.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect::>(); + self.send_http_response(429, headers, Some(response_body.as_bytes())); } fn forward_service_response(&mut self, body_size: usize) { @@ -439,15 +448,8 @@ fn policy_name(policy: Policy) -> &'static str { } } -fn contains_allowed_true(text: &str) -> bool { - text.contains(r#""allowed":true"#) || text.contains(r#""allowed": true"#) -} - -fn extract_json_number(text: &str, key: &str) -> Option { - let needle = format!(r#""{key}":"#); - let pos = text.find(&needle)?; - let digits = text[pos + needle.len()..].chars().skip_while(|c| c.is_whitespace()).take_while(|c| c.is_ascii_digit()).collect::(); - digits.parse().ok() +fn contains_value(values: &[String], needle: &str) -> bool { + values.iter().any(|value| value.eq_ignore_ascii_case(needle)) } fn value_at<'a>(value: &'a Value, path: &[&str]) -> Option<&'a Value> { @@ -502,28 +504,6 @@ fn normalize_header_name(value: &str, fallback: &str) -> String { } } -fn normalize_policy(value: &str) -> Option { - match value.trim().to_ascii_lowercase().as_str() { - "abandon" => Some("abandon".to_string()), - "queue" => Some("queue".to_string()), - "wait" => Some("wait".to_string()), - _ => None, - } -} - -fn normalize_priority(value: &str) -> Option { - match value.trim().to_ascii_lowercase().as_str() { - "high" => Some("high".to_string()), - "normal" | "default" | "medium" => Some("normal".to_string()), - "low" => Some("low".to_string()), - _ => None, - } -} - -fn contains_value(values: &[String], needle: &str) -> bool { - values.iter().any(|value| value.eq_ignore_ascii_case(needle)) -} - #[cfg(test)] mod tests { use super::*; diff --git a/plugins/wasm/ai-gateway-queue/src/policy.rs b/plugins/wasm/ai-gateway-queue/src/policy.rs new file mode 100644 index 00000000..d57ba364 --- /dev/null +++ b/plugins/wasm/ai-gateway-queue/src/policy.rs @@ -0,0 +1,68 @@ +//! 可在 host 侧单测的策略/JSON 纯逻辑（TC-GW / TC-HDR 相关）。 + +/// 规范化 X-RateLimit-Policy 取值。 +pub fn normalize_policy(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "abandon" => Some("abandon".to_string()), + "queue" => Some("queue".to_string()), + "wait" => Some("wait".to_string()), + _ => None, + } +} + +/// 规范化队列优先级 header。 +pub fn normalize_priority(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "high" => Some("high".to_string()), + "normal" | "default" | "medium" => Some("normal".to_string()), + "low" => Some("low".to_string()), + _ => None, + } +} + +/// 解析限流 check 响应中的 allowed=true。 +pub fn contains_allowed_true(text: &str) -> bool { + text.contains(r#""allowed":true"#) || text.contains(r#""allowed": true"#) +} + +/// 从 JSON 文本提取数字字段（如 retry_after_ms）。 +pub fn extract_json_number(text: &str, key: &str) -> Option { + let quoted = format!("\"{key}\""); + for needle in [format!("{quoted}:"), format!("{quoted}:\"")] { + let Some(pos) = text.find(&needle) else { continue }; + let digits = text[pos + needle.len()..] + .chars() + .skip_while(|c| c.is_whitespace() || *c == '"') + .take_while(|c| c.is_ascii_digit()) + .collect::(); + if let Ok(v) = digits.parse() { + return Some(v); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tc_hdr_02_rejects_invalid_policy() { + assert!(normalize_policy("invalid").is_none()); + assert_eq!(normalize_policy("QUEUE").as_deref(), Some("queue")); + } + + #[test] + fn tc_gw_rate_limit_response_parsing() { + assert!(contains_allowed_true(r#"{"allowed":true,"retry_after_ms":0}"#)); + assert!(!contains_allowed_true(r#"{"allowed":false}"#)); + assert_eq!(extract_json_number(r#"{"retry_after_ms":3000}"#, "retry_after_ms"), Some(3000)); + } + + #[test] + fn normalize_priority_values() { + assert_eq!(normalize_priority("HIGH").as_deref(), Some("high")); + assert_eq!(normalize_priority("medium").as_deref(), Some("normal")); + assert!(normalize_priority("urgent").is_none()); + } +} diff --git a/resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.local.json b/resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.local.json new file mode 100644 index 00000000..977ff1ec --- /dev/null +++ b/resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.local.json @@ -0,0 +1,29 @@ +{ + "url": "file:///Users/sh.zhang/Workspace/huayun/jiyan/ai-gateway-dev/spacegate/plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm", + "validate_on_create": false, + "fail_strategy": "fail_close", + "plugin_name": "ai-gateway-queue", + "plugin_root_id": "ai-gateway-queue-root", + "plugin_vm_id": "ai-gateway-queue-vm", + "vm_pool_size": 4, + "wait_vm_pool_size": 4, + "limits": { + "max_memory_pages": 64, + "fuel_per_call": 20000000, + "epoch_timeout_millis": 50, + "max_body_bytes": 33554432, + "max_pending_calls": 1 + }, + "plugin_config": { + "service_cluster": "ai-gateway-service", + "service_authority": "ai-gateway-service", + "rate_limit_path": "/v1/ratelimit/check", + "enqueue_path": "/v1/queue/enqueue", + "wait_path": "/v1/queue/enqueue-and-wait", + "service_timeout_ms": 65000, + "require_policy": true + }, + "clusters": { + "ai-gateway-service": "http://127.0.0.1:18080" + } +} From 0dfdfb8a63ece230e6440c8c081a0b614fb5db24 Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Sun, 24 May 2026 22:13:37 +0800 Subject: [PATCH 16/19] fix: incremental plugin file writes and add admin UI config guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 插件保存改为单文件写入，避免 Docker 共享挂载下清空配置树导致 Read-only/EBUSY；并补充管理界面配置指南与 admin-server 镜像构建说明。 Co-authored-by: Cursor --- crates/config/src/service/fs/create.rs | 17 +- crates/config/src/service/fs/update.rs | 15 +- deploy/README.md | 2 + docs/ai-gateway-queue-admin-ui-guide.md | 524 ++++++++++++++++++++++++ plugins/wasm/ai-gateway-queue/README.md | 3 +- resource/docker/Dockerfile.admin-server | 14 + 6 files changed, 557 insertions(+), 18 deletions(-) create mode 100644 docs/ai-gateway-queue-admin-ui-guide.md create mode 100644 resource/docker/Dockerfile.admin-server diff --git a/crates/config/src/service/fs/create.rs b/crates/config/src/service/fs/create.rs index 8c15b95c..38be51a3 100644 --- a/crates/config/src/service/fs/create.rs +++ b/crates/config/src/service/fs/create.rs @@ -15,14 +15,15 @@ where F: ConfigFormat + Send + Sync, { async fn create_plugin(&self, id: &spacegate_model::PluginInstanceId, value: serde_json::Value) -> Result<(), BoxError> { - self.modify_cached(|config| { - if config.plugins.get(id).is_some() { - return Err("plugin existed".into()); - } - config.plugins.insert(id.clone(), value); - Ok(()) - }) - .await + let path = self.plugin_path(id); + if path.exists() { + return Err("plugin existed".into()); + } + // 仅写入新插件文件，避免 rewrite 整个 /etc/spacegate + tokio::fs::create_dir_all(self.plugin_dir()).await?; + let b_spec = self.format.ser(&value)?; + tokio::fs::write(&path, &b_spec).await?; + Ok(()) } async fn create_config_item(&self, gateway_name: &str, item: ConfigItem) -> Result<(), BoxError> { self.modify_cached(|config| { diff --git a/crates/config/src/service/fs/update.rs b/crates/config/src/service/fs/update.rs index d04bf2c2..e7d34a80 100644 --- a/crates/config/src/service/fs/update.rs +++ b/crates/config/src/service/fs/update.rs @@ -12,15 +12,12 @@ where F: ConfigFormat + Send + Sync, { async fn update_plugin(&self, id: &spacegate_model::PluginInstanceId, value: serde_json::Value) -> Result<(), BoxError> { - self.modify_cached(|config| { - if let Some(prev_spec) = config.plugins.get_mut(id) { - *prev_spec = value; - Ok(()) - } else { - Err("plugin not exists".into()) - } - }) - .await + // 仅更新单个插件 JSON，避免 modify_cached 清空整棵配置树（Docker 共享挂载会 EBUSY/EROFS） + tokio::fs::create_dir_all(self.plugin_dir()).await?; + let path = self.plugin_path(id); + let b_spec = self.format.ser(&value)?; + tokio::fs::write(&path, &b_spec).await?; + Ok(()) } async fn update_config_item_gateway(&self, gateway_name: &str, gateway: SgGateway) -> Result<(), BoxError> { self.modify_cached(|config| { diff --git a/deploy/README.md b/deploy/README.md index f2ebf61b..cdb73ff5 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -5,6 +5,7 @@ 相关文档： - 插件行为与请求头：[`plugins/wasm/ai-gateway-queue/README.md`](../plugins/wasm/ai-gateway-queue/README.md) +- **管理界面配置指南**：[`docs/ai-gateway-queue-admin-ui-guide.md`](../docs/ai-gateway-queue-admin-ui-guide.md) - 测试用例规格：[`docs/ai-gateway-queue-test-spec.md`](../docs/ai-gateway-queue-test-spec.md) - K8s manifest 目录：[`deploy/k8s/ai-gateway/`](k8s/ai-gateway/) @@ -232,6 +233,7 @@ cargo test -p ai-gateway-service --lib 要点： - 配置目录挂载：**工作区根目录** `.docker/ai-gateway-demo/` → 容器内 `/etc/spacegate` +- **admin-server 卷须可写**（勿 `:ro`），否则管理界面保存插件报 `Read-only file system` - Wasm 放在 `plugin/`（JSON）与 `plugins/`（`.wasm` 二进制），**勿**把 `.wasm` 放进 `plugin/`（会被当 JSON 解析导致 SpaceGate 启动失败） - macOS 上 **不能** `docker cp` 本机编译的 Mach-O 二进制进 Linux 容器，需在 Linux 环境构建镜像 diff --git a/docs/ai-gateway-queue-admin-ui-guide.md b/docs/ai-gateway-queue-admin-ui-guide.md new file mode 100644 index 00000000..64f0ffe9 --- /dev/null +++ b/docs/ai-gateway-queue-admin-ui-guide.md @@ -0,0 +1,524 @@ +# AI 网关排队限流插件 — 管理界面配置指南 + +本文说明如何在 **SpaceGate Admin 管理界面** 中配置 **AI 请求队列网关**（`ai-gateway-queue` Wasm 插件），包括插件实例创建、网关/路由挂载、租户配额，以及客户端请求头约定。 + +相关文档： + +- 插件行为与 API：[`plugins/wasm/ai-gateway-queue/README.md`](../plugins/wasm/ai-gateway-queue/README.md) +- 编译与 K8s 部署：[`deploy/README.md`](../deploy/README.md) +- 测试用例：[`ai-gateway-queue-test-spec.md`](ai-gateway-queue-test-spec.md) + +--- + +## 1. 配置全景 + +管理界面上的配置分 **三层**，需按顺序完成： + +```text +┌─────────────────────────────────────────────────────────────┐ +│ ① 插件实例（插件页 → AI 请求队列网关） │ +│ 写入 plugin/wasm.ai-gateway-queue.json │ +│ 含 Wasm URL、后端地址、plugin_config 等 │ +└───────────────────────────┬─────────────────────────────────┘ + │ +┌───────────────────────────▼─────────────────────────────────┐ +│ ② 挂载引用（网关页 或 路由页 → 插件列表） │ +│ 仅引用 { code: wasm, name: ai-gateway-queue } │ +│ ⚠ 只选一层挂载，勿 Gateway + Route 重复 │ +└───────────────────────────┬─────────────────────────────────┘ + │ +┌───────────────────────────▼─────────────────────────────────┐ +│ ③ 租户配额（ai-gateway-service Admin API） │ +│ UI「队列配额」Tab 当前为占位；可用 API 或 curl 配置 │ +└─────────────────────────────────────────────────────────────┘ +``` + +| 层级 | 管理界面入口 | 落盘 / 存储 | +|------|-------------|-------------| +| 插件实例 | **插件** → Tab **AI** → **AI 请求队列网关** | `plugin/wasm.ai-gateway-queue.json` | +| 挂载引用 | **网关** 或 **路由** → 插件列表 | `gateway/{name}/config.json` 或 `route/{route}.json` | +| 租户配额 | API（见 §6） | Redis | + +--- + +## 2. 前置条件 + +### 2.1 依赖服务 + +| 服务 | 默认端口 | 说明 | +|------|---------|------| +| **spacegate-admin-server** | 9992（开发）/ 9080（Docker 管理端） | 读写 SpaceGate 配置 | +| **spacegate-admin-fe** | 4000 | Vue 管理界面 | +| **SpaceGate 网关** | 9993 | 加载 Wasm 并转发流量 | +| **ai-gateway-service** | 18080 | 限流 / 入队 / Worker 后端 | +| **Redis 7+** | 6379 | 令牌桶与队列 | +| **上游 LLM** | 9000（示例） | HTTPRoute 后端 | + +### 2.2 启动管理界面（本地开发） + +```bash +# 终端 1：Admin 后端（文件配置模式示例） +cd spacegate +cargo run -p spacegate-admin-server -- -c file:.docker/ai-gateway-demo + +# 终端 2：Admin 前端 +cd spacegate-admin-fe +npm install +npm run dev +# 浏览器打开 http://localhost:4000 +``` + +Docker 环境可直接访问 **`http://localhost:9080`**（`ai-gateway-web` 容器）。 + +### 2.3 配置 ai-gateway-service 地址（重要） + +插件 Drawer 中的 **Schema 表单**、**文档 Tab** 以及未来的 **租户配额** 均通过 `ai-gateway-service` 的 Admin API 拉取： + +```bash +# spacegate-admin-fe/.env.local（或构建时环境变量） +VITE_AI_GATEWAY_BASE_URL=http://127.0.0.1:18080 +``` + +| 是否配置 | 效果 | +|---------|------| +| **已配置** | Schema / Readme 正常加载；租户配额 API 可用 | +| **未配置** | 请求打到前端 `:4000`，Schema 加载失败，表单为空 | + +SpaceGate 配置 API（保存插件、网关、路由）走 `/api` 代理到 admin-server，**与上述变量无关**。 + +本地 Vite 代理（`vite.config.ts`）： + +```text +/api/* → http://localhost:9992/* +``` + +--- + +## 3. 界面导航 + +### 3.1 选择网关 + +顶部 **SelectGateway** 下拉框选择目标网关（如 `ai-demo`）。后续菜单跳转会自动带上 `?gatewayName=ai-demo`。 + +### 3.2 左侧菜单 + +| 菜单 | 路径 | 与本插件相关用途 | +|------|------|-----------------| +| **网关** | `/gateway` | 网关级插件挂载、监听器 | +| **路由** | `/route` | 路由规则、后端、规则级插件 | +| **插件** | `/plugins` | **创建 / 编辑 AI 请求队列网关实例** | +| **实例** | `/instance` | SpaceGate 进程在线状态（与插件配置无关） | + +--- + +## 4. 分步配置 + +### 步骤 1：创建插件实例 + +1. 进入 **插件** 页 +2. 切换到 Tab **「AI」** +3. 找到卡片 **「AI 请求队列网关」** +4. 点击 **「配置」**，打开 **AI 请求队列网关** Drawer +5. 填写 **基础接入** 与 **基础配置**（见 §5） +6. 点击 **保存** + +保存后 admin-server 写入： + +```text +plugin/wasm.ai-gateway-queue.json +``` + +首次保存调用 `POST /config/plugin`；再次编辑调用 `PUT /config/plugin`。 + +卡片上会显示 **「已部署」** 标签。 + +### 步骤 2：挂载到网关或路由 + +插件实例创建后，还需在 **网关** 或 **路由** 中引用，流量才会经过 Wasm。 + +#### 方式 A：网关级挂载（推荐） + +1. 进入 **网关** 页 +2. 编辑目标网关（如 `ai-demo`） +3. 找到 **插件** 字段（PluginListForm） +4. 点击 **添加插件** +5. 选择： + - **Code**：`wasm` + - **Kind**：`named` + - **Name**：`ai-gateway-queue` +6. 保存网关配置 + +等价 JSON 片段： + +```json +{ + "plugins": [ + { + "code": "wasm", + "kind": "named", + "name": "ai-gateway-queue" + } + ] +} +``` + +#### 方式 B：路由级挂载 + +1. 进入 **路由** 页 +2. 编辑目标路由（如 `ai`）下的某条 **规则** +3. 在规则 **插件** 列表中添加同样的引用 +4. 配置 **后端** 指向 LLM 上游 +5. 保存 + +等价 JSON 片段（规则内）： + +```json +{ + "matches": [{ "path": { "kind": "Prefix", "value": "/v1/" } }], + "plugins": [ + { "code": "wasm", "kind": "named", "name": "ai-gateway-queue" } + ], + "backends": [{ "host": { "kind": "Host", "host": "127.0.0.1" }, "port": 9000, "weight": 1 }] +} +``` + +> **⚠ 切勿重复挂载** +> 若 Gateway 与 Route **同时** 引用 `ai-gateway-queue`，每个请求会执行 **两次** 插件逻辑，导致 **双倍扣 token / 双倍限流**。 +> 生产环境请 **只选一层**；`resource/ai-gateway-demo` 示例为演示方便两处都挂了，本地验证时注意这一点。 + +### 步骤 3：配置路由与后端 + +在 **路由** 页确保： + +- 路径匹配 AI API（如 `/v1/` Prefix） +- **后端** 指向真实 LLM 服务地址与端口 +- 优先级（priority）按需设置 + +### 步骤 4：配置租户配额（可选） + +按租户 / 模型 / 路径 / 策略设置差异化令牌桶，见 **§6**。当前 Drawer 内 **「队列配额」Tab 为占位**，需通过 API 配置。 + +### 步骤 5：验证 + +```bash +# 经网关（插件生效） +curl -i http://127.0.0.1:9993/v1/chat/completions \ + -H 'X-RateLimit-Policy: abandon' \ + -H 'X-Tenant-Id: demo' \ + -H 'Content-Type: application/json' \ + -d '{"prompt":"hello"}' + +# 直连后端健康检查 +curl http://127.0.0.1:18080/healthz +``` + +期望：配额内 `200`；缺 Policy 且 `require=true` 时 `400`；超额 abandon `429`。 + +--- + +## 5. Drawer 字段说明 + +打开 **插件 → AI → AI 请求队列网关 → 配置** 后，Drawer 含四个 Tab。 + +### 5.1 Tab「基础配置」 + +#### 基础接入（Wasm 宿主层 → `spec` 顶层） + +| 界面字段 | 配置键 | 默认值 | 说明 | +|---------|--------|--------|------| +| Wasm URL | `url` | 空 | Wasm 制品地址。支持 `file://`、`http(s)://`、`oci://` | +| 插件名称 | `plugin_name` | `ai-gateway-queue` | 建议保持不变 | +| 失败策略 | `fail_strategy` | `fail_close` | `fail_close`：插件异常时拒绝请求；`fail_open`：放行 | +| 队列后端地址 | `clusters["ai-gateway-service"]` | `http://127.0.0.1:18080` | ai-gateway-service 的 HTTP 地址 | +| 普通 VM 池大小 | `vm_pool_size` | `4` | 处理 abandon / queue 短请求的 Wasm 实例数，≥1 | +| Wait VM 池大小 | `wait_vm_pool_size` | `4` | wait 长连接专用池；不用 wait 可设 `0` | + +**Wasm URL 示例：** + +| 环境 | 示例值 | +|------|--------| +| 本地 Cargo | `file:///path/to/spacegate/plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm` | +| Docker 挂载 | `file:///etc/spacegate/plugins/spacegate_plugin_ai_gateway_queue.wasm` | +| K8s HTTP 分发 | `http://ai-gateway-wasm/spacegate_plugin_ai_gateway_queue.wasm` | +| OCI 制品 | `oci://ghcr.io/your-org/ai-gateway-queue:v1.0.0` | + +**界面未暴露、保存时会保留的字段**（来自已有配置文件）： + +- `validate_on_create`、`plugin_root_id`、`plugin_vm_id` +- `limits`：`max_memory_pages`、`fuel_per_call`、`epoch_timeout_millis`、`max_body_bytes`、`max_pending_calls` + +#### 基础配置 Schema 表单（→ `spec.plugin_config`） + +表单字段由 `ai-gateway-service` 动态提供：`GET /v1/admin/plugins/ai-gateway-queue/schema`。 + +##### service — 队列后端接入 + +| 字段 | 默认 | 说明 | +|------|------|------| +| `cluster` | `ai-gateway-service` | SpaceGate cluster 名，须与 `clusters` 键一致 | +| `authority` | `ai-gateway-service` | HTTP 调用的 `:authority` | +| `timeout_ms` | `65000` | 调用后端超时；使用 wait 模式建议 ≥60000 | + +##### paths — 后端 API 路径 + +| 字段 | 默认 | +|------|------| +| `rate_limit` | `/v1/ratelimit/check` | +| `enqueue` | `/v1/queue/enqueue` | +| `wait` | `/v1/queue/enqueue-and-wait` | + +一般保持默认即可，除非后端改了路由前缀。 + +##### headers — 客户端请求头映射 + +| 字段 | 默认 HTTP 头 | 用途 | +|------|-------------|------| +| `policy` | `X-RateLimit-Policy` | 队列策略 | +| `tenant` | `X-Tenant-Id` | 租户标识 | +| `model` | `X-Model` | 模型名（优先级路由） | +| `priority` | `X-Queue-Priority` | 显式优先级 | + +HTTP 头名大小写不敏感；配置中通常写小写。 + +##### policies — 策略校验 + +| 字段 | 默认 | 说明 | +|------|------|------| +| `require` | `true` | 为 `true` 时，缺少 Policy 头 → **400** | +| `default` | 空 | `require=false` 时使用的默认策略：`abandon` / `queue` / `wait` | + +##### priority — 多优先级队列 + +| 字段 | 默认 | 说明 | +|------|------|------| +| `enabled` | `true` | 关闭后所有请求走 `default` 优先级 | +| `default` | `normal` | `high` / `normal` / `low` | +| `high_models` / `low_models` | `[]` | 模型名精确匹配 | +| `high_tenants` / `low_tenants` | `[]` | 租户 ID 列表 | + +> **扁平 vs 嵌套格式** +> 部分示例文件（如 `resource/ai-gateway-demo`）使用扁平键（`service_cluster`、`require_policy`）。 +> 管理界面 SchemaForm 使用 **嵌套 JSON**。Wasm 运行时两种格式均兼容；若从文件导入后表单显示异常，可在 Drawer 中重新保存一次以统一格式。 + +### 5.2 Tab「队列配额」 + +当前版本显示占位说明：**租户差异化限流 UI 尚未接入 Drawer**。 + +V1 行为说明（与界面提示一致）： + +- **全局限流**在 `ai-gateway-service` 配置，非 Drawer 字段 +- 环境变量：`AI_RATE_LIMIT_RPS`、`AI_RATE_LIMIT_BURST`、`AI_RATE_LIMIT_COST` +- 或 TOML `[rate_limit]` 段 + +租户级配额请使用 **§6 API**。 + +### 5.3 Tab「文档」 + +从 `GET /v1/admin/plugins/ai-gateway-queue/readme` 拉取插件 README Markdown，便于在界面内查阅行为说明。 + +### 5.4 Tab「队列观测」 + +V1 预留，后续接入队列长度、消费速率、回调失败等指标。 + +--- + +## 6. 租户配额配置（Admin API） + +`TenantRateLimitTable` 组件已实现完整 CRUD，但尚未挂接到 Drawer「队列配额」Tab。可通过 HTTP API 或 curl 配置。 + +### 6.1 创建 / 更新配额 + +```bash +curl -X PUT http://127.0.0.1:18080/v1/admin/tenant-rate-limits \ + -H 'Content-Type: application/json' \ + -d '{ + "tenant": "demo", + "model": "", + "path": "", + "policy": "", + "rps": 10, + "burst": 20, + "cost": 1 + }' +``` + +### 6.2 字段说明 + +| 字段 | 必填 | 默认 | 说明 | +|------|------|------|------| +| `tenant` | 是 | — | 租户 ID，与 `X-Tenant-Id` 对应 | +| `model` | 否 | 空=通配 | 如 `gpt-4o` | +| `path` | 否 | 空=通配 | 如 `/v1/chat/completions` | +| `policy` | 否 | 空=通配 | `abandon` / `queue` / `wait` | +| `rps` | 是 | — | 每秒令牌恢复速率，>0 | +| `burst` | 是 | — | 突发容量（令牌桶大小），>0 | +| `cost` | 是 | 1 | 单次请求消耗令牌数，>0 | +| `ttl_secs` | 否 | 永久 | 临时配额过期秒数 | + +**匹配优先级**：维度越具体越优先（带 `model+path+policy` 的规则优先于仅 `tenant` 的规则）。 + +Redis key 预览格式： + +```text +ai:tenant:ratelimit:{tenant}[:model:...][:path:...][:policy:...] +``` + +### 6.3 查询与删除 + +```bash +# 列表（可按 tenant 过滤） +curl 'http://127.0.0.1:18080/v1/admin/tenant-rate-limits?tenant=demo' + +# 删除（body 与创建时维度一致） +curl -X DELETE http://127.0.0.1:18080/v1/admin/tenant-rate-limits \ + -H 'Content-Type: application/json' \ + -d '{"tenant":"demo","rps":10,"burst":20,"cost":1}' +``` + +--- + +## 7. 客户端请求头 + +配置完成后，调用方经网关 `:9993` 发送请求时需携带： + +| 请求头 | 必填 | 取值 | 说明 | +|--------|------|------|------| +| `X-RateLimit-Policy` | 当 `require=true` | `abandon` / `queue` / `wait` | 必须小写 | +| `X-Tenant-Id` | 建议 | 任意字符串 | 租户隔离与配额匹配 | +| `X-Callback-URL` | queue 超额时 | HTTPS URL | 异步回调地址 | +| `X-Model` | 否 | 模型名 | 影响优先级路由 | +| `X-Queue-Priority` | 否 | `high` / `normal` / `low` | 显式优先级 | + +**三种策略行为（均需先过令牌桶）：** + +| 策略 | 配额内 | 超额 | +|------|--------|------| +| `abandon` | 直通上游 200 | 429，不入队 | +| `queue` | 直通上游 200 | 202 + job_id，回调/轮询取结果 | +| `wait` | 直通上游 200 | 阻塞等待结果，超时 504 | + +示例： + +```bash +# abandon — 超额返回 429 +curl -i http://127.0.0.1:9993/v1/chat/completions \ + -H 'X-RateLimit-Policy: abandon' \ + -H 'X-Tenant-Id: demo' \ + -H 'Content-Type: application/json' \ + -d '{"prompt":"hi"}' + +# queue — 超额返回 202 +curl -i http://127.0.0.1:9993/v1/chat/completions \ + -H 'X-RateLimit-Policy: queue' \ + -H 'X-Tenant-Id: demo' \ + -H 'X-Callback-URL: https://example.com/callback' \ + -H 'Content-Type: application/json' \ + -d '{"prompt":"hi"}' +``` + +--- + +## 8. 配置与存储映射 + +```text +管理界面操作 API 存储位置 +───────────────────────────────────────────────────────────────────────── +保存 AI 队列 Drawer POST/PUT /config/plugin plugin/wasm.ai-gateway-queue.json +保存网关 PUT /config/item/{gw}/gateway gateway/{gw}/config.json +保存路由规则 PUT .../route/item/{route} gateway/{gw}/route/{route}.json +租户配额 PUT PUT /v1/admin/tenant-rate-limits Redis +读取 Schema GET /v1/admin/plugins/.../schema （运行时生成） +``` + +文件命名规则：`{code}.{name}.json` → `wasm.ai-gateway-queue.json`。 + +--- + +## 9. 常见问题 + +### Q1：Schema 表单空白或加载失败？ + +检查 `VITE_AI_GATEWAY_BASE_URL` 是否指向运行中的 `ai-gateway-service`（默认 `http://127.0.0.1:18080`），并确认 `/healthz` 可访问。 + +### Q2：保存插件成功但请求未限流？ + +1. 是否在 **网关或路由** 中添加了插件引用？ +2. 是否 **重复挂载** 导致行为异常？ +3. SpaceGate 是否已加载最新配置（文件模式通常自动热更）？ + +### Q3：第一次请求就 429？ + +- 检查 Gateway + Route **双重挂载** +- 检查租户 `burst` 是否过小 +- 用 Admin API 调高配额或新建租户规则 + +### Q4：缺 Policy 返回 400？ + +`plugin_config.policies.require=true`（默认）。客户端必须带 `X-RateLimit-Policy`，或在 Drawer 中关闭 require 并设置 default。 + +### Q5：保存插件配置报 `Read-only file system (os error 30)`？ + +**原因：** Docker 队列模式下 `admin-server` 配置卷被挂成 **只读（`:ro`）**，无法写入 `plugin/wasm.ai-gateway-queue.json`。 + +**修复：** + +1. `docker-compose.queue.yml` 中 admin-server 使用 **整目录可写** 挂载（勿 `:ro`）： + +```yaml +admin-server: + volumes: + - ./.docker/ai-gateway-demo:/etc/spacegate +``` + +2. Wasm 二进制挂到配置目录外，URL 用 `file:///opt/wasm/...`（见 `docker-compose.queue.yml` 注释）。 + +3. 重建 admin-server 镜像（含插件增量写入修复）后重启容器： + +```bash +cd spacegate +docker build -f resource/docker/Dockerfile.admin-server -t ai-gateway/admin-server:dev . +docker rm -f ai-gateway-admin-server && docker run -d --name ai-gateway-admin-server \ + --network container:ai-gateway-spacegate --restart unless-stopped \ + -e CONFIG=file:/etc/spacegate -e RUST_LOG=info \ + -v $(pwd)/../.docker/ai-gateway-demo:/etc/spacegate \ + ai-gateway/admin-server:dev -c file:/etc/spacegate -p 19992 -H 0.0.0.0 +``` + +**临时绕过：** 直接编辑宿主机 `.docker/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json`，无需走 UI 保存。 + +### Q6：`:9080` 管理端报 No such file or directory？ + +admin-server 读不到 `/etc/spacegate` 配置。Docker 环境检查 **工作区根目录** `.docker/ai-gateway-demo` 是否正确挂载到容器内 `/etc/spacegate`。 + +### Q7：K8s 环境能用这套 UI 吗？ + +可以管理 SpaceGate 配置（若 admin-server 连到同一配置源）。K8s 下 Wasm 常通过 **SgFilter** 内联 spec + HTTP/OCI 分发 Wasm，详见 [`deploy/README.md`](../deploy/README.md) §6。Higress **WasmPlugin** CR 的 `defaultConfig` **不含** `clusters`，生产建议用 **SgFilter**。 + +--- + +## 10. 推荐配置流程（ checklist ） + +```text +□ Redis、ai-gateway-service、上游 LLM 已启动 +□ 编译 Wasm 并确认 url 可访问 +□ 设置 VITE_AI_GATEWAY_BASE_URL +□ 插件页 → AI → 配置 AI 请求队列网关 → 保存 +□ 网关或路由（二选一）添加 wasm / ai-gateway-queue 引用 +□ 路由后端指向 LLM 服务 +□ （可选）PUT /v1/admin/tenant-rate-limits 配置租户配额 +□ curl 冒烟：400（无 Policy）/ 200（配额内）/ 429（超额 abandon） +``` + +--- + +## 11. 相关源码索引 + +| 文件 | 说明 | +|------|------| +| `spacegate-admin-fe/components/config/src/components/PluginPanel.vue` | AI Tab 与 Drawer 入口 | +| `spacegate-admin-fe/components/config/src/components/AiGatewayQueueDrawer.vue` | 主配置 Drawer | +| `spacegate-admin-fe/components/config/src/components/TenantRateLimitTable.vue` | 租户配额表格（待接入 Tab） | +| `spacegate-admin-fe/components/config/src/api/aiGateway.ts` | ai-gateway-service Admin API 客户端 | +| `binary/ai-gateway-service/src/app/admin.rs` | Schema / Readme 端点 | +| `binary/ai-gateway-service/src/app/types.rs` | `AiGatewayQueuePluginConfig` 结构 | +| `resource/ai-gateway-demo/` | 文件模式配置模板 | diff --git a/plugins/wasm/ai-gateway-queue/README.md b/plugins/wasm/ai-gateway-queue/README.md index eb2cbd52..5045bda3 100644 --- a/plugins/wasm/ai-gateway-queue/README.md +++ b/plugins/wasm/ai-gateway-queue/README.md @@ -85,7 +85,8 @@ AI_REQUIRE_HTTPS_CALLBACK=false 可参考： -`/Users/sh.zhang/Workspace/huayun/jiyan/ai-gateway-dev/spacegate/resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json` +- 文件模板：[`resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json`](../../resource/ai-gateway-demo/plugin/wasm.ai-gateway-queue.json) +- **管理界面操作步骤**：[`docs/ai-gateway-queue-admin-ui-guide.md`](../../docs/ai-gateway-queue-admin-ui-guide.md) 关键配置项： diff --git a/resource/docker/Dockerfile.admin-server b/resource/docker/Dockerfile.admin-server new file mode 100644 index 00000000..fa29b9ed --- /dev/null +++ b/resource/docker/Dockerfile.admin-server @@ -0,0 +1,14 @@ +# 仅重建 spacegate-admin-server（含插件增量写入修复） +FROM rust:1.85-bookworm AS builder +WORKDIR /app +COPY . . +RUN cargo build --release -p spacegate-admin-server + +FROM debian:bookworm-slim +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates tini \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /app/target/release/spacegate-admin-server /usr/local/bin/spacegate-admin-server +ENV CONFIG=file:/etc/spacegate RUST_LOG=info +EXPOSE 19992 +ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/spacegate-admin-server"] +CMD ["-c", "file:/etc/spacegate", "-p", "19992", "-H", "0.0.0.0"] From 31048077cea5614f9bb75efd40ca4337937de155 Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Sun, 24 May 2026 23:31:44 +0800 Subject: [PATCH 17/19] feat: add Docker full-stack large body E2E script Wire run-gateway-e2e.sh to the gateway large-body test and bump admin-server Docker build to Rust 1.88. Co-authored-by: Cursor --- .../scripts/run-gateway-e2e.sh | 8 +- .../scripts/run-gateway-large-body-e2e.sh | 108 ++++++++++++++++++ resource/docker/Dockerfile.admin-server | 2 +- 3 files changed, 112 insertions(+), 6 deletions(-) create mode 100755 binary/ai-gateway-service/scripts/run-gateway-large-body-e2e.sh diff --git a/binary/ai-gateway-service/scripts/run-gateway-e2e.sh b/binary/ai-gateway-service/scripts/run-gateway-e2e.sh index 3be23874..16a4a6b8 100755 --- a/binary/ai-gateway-service/scripts/run-gateway-e2e.sh +++ b/binary/ai-gateway-service/scripts/run-gateway-e2e.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -# 可选：SpaceGate + Wasm 全链路 E2E（TC-GW-*） -# 需已编译 wasm 且 SpaceGate demo 配置就绪；默认 skip。 +# SpaceGate + Wasm 全链路 E2E（TC-GW-* / TC-GW-BODY-*） set -euo pipefail -echo "TC-GW-* gateway E2E: optional — configure SpaceGate demo and run manual curl." -echo "See spacegate/docs/ai-gateway-queue-test-spec.md section TC-GW." -exit 0 +DIR="$(cd "$(dirname "$0")" && pwd)" +exec "$DIR/run-gateway-large-body-e2e.sh" "$@" diff --git a/binary/ai-gateway-service/scripts/run-gateway-large-body-e2e.sh b/binary/ai-gateway-service/scripts/run-gateway-large-body-e2e.sh new file mode 100755 index 00000000..9fffcfd8 --- /dev/null +++ b/binary/ai-gateway-service/scripts/run-gateway-large-body-e2e.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# TC-GW-BODY-01 / TC-BODY-02：Docker 全栈大 body E2E +# 流量路径：Client -> SpaceGate(:9993) Wasm -> ai-gateway-service -> MinIO + Redis +# +# 前置： +# docker compose -f docker-compose.yml -f docker-compose.queue.yml --profile queue up -d +# ./scripts/sync-wasm-plugin-to-docker-config.sh && 重启 spacegate +# +# 用法： +# ./spacegate/binary/ai-gateway-service/scripts/run-gateway-large-body-e2e.sh +# ENSURE_STACK=1 ./spacegate/.../run-gateway-large-body-e2e.sh # 自动 compose up +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../../.." && pwd)" +cd "$ROOT" + +GATEWAY="${GATEWAY_URL:-http://127.0.0.1:9993}" +SERVICE="${SERVICE_URL:-http://127.0.0.1:18080}" +MINIO_HOST="${MINIO_HOST:-http://127.0.0.1:9010}" +MINIO_USER="${MINIO_ROOT_USER:-minioadmin}" +MINIO_PASS="${MINIO_ROOT_PASSWORD:-minioadmin}" +BUCKET="${AI_OBJECT_STORE_BUCKET:-ai-gateway-body}" +# 200 KiB，超过默认 inline 阈值 128 KiB +BODY_SIZE="${LARGE_BODY_BYTES:-204800}" +TENANT="gw-large-body-${RANDOM}" +COMPOSE=(docker compose -f docker-compose.yml -f docker-compose.queue.yml --profile queue) + +die() { echo "ERROR: $*" >&2; exit 1; } + +metric_value() { + local name="$1" + curl -sf "$SERVICE/metrics" | awk -v n="$name" '$1 == n { print $2; exit }' +} + +wait_http() { + local url="$1" retries="${2:-30}" + for _ in $(seq 1 "$retries"); do + curl -sf "$url" >/dev/null 2>&1 && return 0 + sleep 1 + done + return 1 +} + +if [[ "${ENSURE_STACK:-0}" == "1" ]]; then + echo "==> 启动 / 更新 Docker 栈（queue profile）" + export DOCKER_BUILDKIT=1 + if [[ -x ./scripts/sync-wasm-plugin-to-docker-config.sh ]]; then + ./scripts/sync-wasm-plugin-to-docker-config.sh + fi + "${COMPOSE[@]}" up -d --build minio minio-init ai-gateway-service spacegate admin-server +fi + +echo "==> 前置检查" +wait_http "$SERVICE/healthz" || die "ai-gateway-service 不可达: $SERVICE" +wait_http "http://127.0.0.1:19880/health" || die "SpaceGate 不可达" +curl -sf "$MINIO_HOST/minio/health/live" >/dev/null || die "MinIO 不可达: $MINIO_HOST" + +echo "==> 配置租户限流（burst=1，便于触发 queue 超额入队）" +curl -sf -X PUT "$SERVICE/v1/admin/tenant-rate-limits" \ + -H 'Content-Type: application/json' \ + -d "{\"tenant\":\"$TENANT\",\"rps\":1,\"burst\":1}" >/dev/null + +BASELINE="$(metric_value object_offload_total || echo 0)" + +echo "==> 消耗令牌（abandon 配额内 1 次）" +code=$(curl -s -o /dev/null -w '%{http_code}' -X POST "$GATEWAY/v1/chat/completions" \ + -H 'X-RateLimit-Policy: abandon' \ + -H "X-Tenant-Id: $TENANT" \ + -H 'Content-Type: application/json' \ + -d '{"warmup":true}') +[[ "$code" == "200" ]] || die "预热请求期望 200，实际 $code" + +echo "==> 经网关发送大 body（queue 策略，应 202 入队）" +LARGE="$(python3 -c "print('x'*${BODY_SIZE})")" +# 回调走 Docker 内 mock-upstream，service 容器可直接访问 +CALLBACK="${CALLBACK_URL:-http://mock-upstream:9000/callback}" + +http=$(curl -sS -o /tmp/gw-large-body.json -w '%{http_code}' \ + -X POST "$GATEWAY/v1/chat/completions" \ + -H "X-Tenant-Id: $TENANT" \ + -H 'X-RateLimit-Policy: queue' \ + -H "X-Callback-URL: $CALLBACK" \ + -H 'Content-Type: application/octet-stream' \ + --data-binary "$LARGE") +[[ "$http" == "202" ]] || die "大 body 入队期望 202，实际 $http body=$(cat /tmp/gw-large-body.json)" + +echo "==> 等待 object_offload_total 递增" +found=0 +for _ in $(seq 1 45); do + now="$(metric_value object_offload_total || echo 0)" + if awk -v a="$BASELINE" -v b="$now" 'BEGIN{exit !(b>a)}'; then + echo "object_offload_total: $BASELINE -> $now" + found=1 + break + fi + sleep 1 +done +[[ "$found" == "1" ]] || die "object_offload_total 未递增（baseline=$BASELINE）" + +echo "==> 验证 MinIO bucket 内有对象" +obj_count=$(docker run --rm --network ai-gateway-net --entrypoint /bin/sh minio/mc:latest \ + -c " + mc alias set local http://minio:9000 '$MINIO_USER' '$MINIO_PASS' >/dev/null && + mc ls -r local/$BUCKET/bodies 2>/dev/null | wc -l | tr -d ' ' + ") +[[ "${obj_count:-0}" -gt 0 ]] || die "MinIO bucket/$BUCKET 下未发现 bodies/ 对象" + +echo "==> 全栈大 body E2E 通过（tenant=$TENANT, body=${BODY_SIZE}B, minio_objects>=1）" diff --git a/resource/docker/Dockerfile.admin-server b/resource/docker/Dockerfile.admin-server index fa29b9ed..4a598229 100644 --- a/resource/docker/Dockerfile.admin-server +++ b/resource/docker/Dockerfile.admin-server @@ -1,5 +1,5 @@ # 仅重建 spacegate-admin-server（含插件增量写入修复） -FROM rust:1.85-bookworm AS builder +FROM rust:1.88-bookworm AS builder WORKDIR /app COPY . . RUN cargo build --release -p spacegate-admin-server From 9dad47b71ba79be9b2d9356b146aabf8766fc6fd Mon Sep 17 00:00:00 2001 From: yiye <410033402@qq.com> Date: Mon, 25 May 2026 10:48:06 +0800 Subject: [PATCH 18/19] feat(observability): add OTLP telemetry and audit access logs --- .gitignore | 4 +- Cargo.toml | 6 + binary/spacegate/Cargo.toml | 2 - binary/spacegate/src/main.rs | 2 - crates/config/src/service.rs | 1 + crates/config/src/service/fs/mod.rs | 13 +- crates/config/src/service/fs/model.rs | 7 +- .../service/k8s/convert/gateway_k8s_conv.rs | 58 +- crates/config/tests/test_k8s_config.rs | 58 +- crates/kernel/Cargo.toml | 3 + crates/kernel/src/extension.rs | 2 + crates/kernel/src/extension/route_name.rs | 18 + crates/kernel/src/lib.rs | 2 + crates/kernel/src/observability.rs | 503 ++++++++++++++ crates/kernel/src/service.rs | 200 +++++- crates/kernel/src/service/http_gateway.rs | 13 +- crates/kernel/src/service/http_route.rs | 1 + crates/model/src/constants.rs | 10 + crates/model/src/gateway.rs | 3 +- crates/model/src/lib.rs | 5 + crates/model/src/observability.rs | 109 ++++ crates/model/tests/test_parse_config.rs | 131 ++++ crates/plugin-wasm/Cargo.toml | 1 + crates/plugin-wasm/src/shared.rs | 36 +- crates/plugin/src/lib.rs | 15 + crates/plugin/src/plugins/limit.rs | 2 +- crates/plugin/tests/test_telemetry.rs | 38 ++ crates/shell/Cargo.toml | 7 +- crates/shell/src/config.rs | 1 + crates/shell/src/lib.rs | 2 + crates/shell/src/observability.rs | 224 +++++++ crates/shell/src/server.rs | 4 +- docs/otlp/otel-three-signals-guide.md | 445 +++++++++++++ docs/otlp/telemetry-pluginized-audit-plan.md | 615 ++++++++++++++++++ .../otlp/wasm-telemetry-host-function-plan.md | 251 +++++++ 35 files changed, 2746 insertions(+), 46 deletions(-) create mode 100644 crates/kernel/src/extension/route_name.rs create mode 100644 crates/kernel/src/observability.rs create mode 100644 crates/model/src/observability.rs create mode 100644 crates/plugin/tests/test_telemetry.rs create mode 100644 crates/shell/src/observability.rs create mode 100644 docs/otlp/otel-three-signals-guide.md create mode 100644 docs/otlp/telemetry-pluginized-audit-plan.md create mode 100644 docs/otlp/wasm-telemetry-host-function-plan.md diff --git a/.gitignore b/.gitignore index b6c0d3b1..d0da7b4d 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,6 @@ util/gh-pages/lints.json # dev script devsh/* -.uuid \ No newline at end of file +.uuid +scripts +mix \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index d507202b..a9ba6181 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,12 @@ toml = { version = "0.8", features = ["preserve_order"] } lazy_static = { version = "1.4" } tracing-subscriber = { version = "0.3", features = ["env-filter"] } tracing = { version = "0" } +tracing-opentelemetry = { version = "0.33" } +opentelemetry = { version = "0.32" } +opentelemetry_sdk = { version = "0.32" } +opentelemetry-otlp = { version = "0.32", features = ["grpc-tonic", "http-proto", "reqwest-blocking-client"] } +opentelemetry-appender-tracing = { version = "0.32" } +opentelemetry-semantic-conventions = { version = "0.32" } # Encode base64 = { version = "0.22" } diff --git a/binary/spacegate/Cargo.toml b/binary/spacegate/Cargo.toml index 3ac59171..911eadf6 100644 --- a/binary/spacegate/Cargo.toml +++ b/binary/spacegate/Cargo.toml @@ -36,8 +36,6 @@ serde = { workspace = true, features = ["derive"] } spacegate-shell = { workspace = true } openssl = { version = "0.10" } # tardis = { workspace = true, features = ["console-subscriber"] } -# tardis = { workspace = true } -tracing-subscriber = { workspace = true, features = ["env-filter"] } tokio = { version = "1", features = ["full"] } [dev-dependencies] diff --git a/binary/spacegate/src/main.rs b/binary/spacegate/src/main.rs index 12d177b4..32be3427 100644 --- a/binary/spacegate/src/main.rs +++ b/binary/spacegate/src/main.rs @@ -2,8 +2,6 @@ use clap::Parser; use spacegate_shell::BoxError; mod args; fn main() -> Result<(), BoxError> { - // TODO: more subscriber required - tracing_subscriber::fmt().with_env_filter(tracing_subscriber::EnvFilter::from_default_env()).init(); let args = args::Args::parse(); #[allow(unused_variables)] if let Some(plugins) = args.plugins { diff --git a/crates/config/src/service.rs b/crates/config/src/service.rs index eae9eb5f..d942fc69 100644 --- a/crates/config/src/service.rs +++ b/crates/config/src/service.rs @@ -134,6 +134,7 @@ pub trait Retrieve: Sync + Send { gateways, plugins: PluginInstanceMap::from_config_vec(plugins), api_port: None, + observability: Default::default(), }) } } diff --git a/crates/config/src/service/fs/mod.rs b/crates/config/src/service/fs/mod.rs index e839e811..6e7a362b 100644 --- a/crates/config/src/service/fs/mod.rs +++ b/crates/config/src/service/fs/mod.rs @@ -99,8 +99,17 @@ where pub async fn save_config(&self, config: Config) -> Result<(), BoxError> { // save config - let Config { plugins, gateways, api_port } = config; - let main_config_to_save: Config = Config { api_port, ..Default::default() }; + let Config { + plugins, + gateways, + api_port, + observability, + } = config; + let main_config_to_save: Config = Config { + api_port, + observability, + ..Default::default() + }; let b_main_config = self.format.ser(&main_config_to_save)?; tokio::fs::write(self.entrance_config_path(), &b_main_config).await?; if !plugins.is_empty() { diff --git a/crates/config/src/service/fs/model.rs b/crates/config/src/service/fs/model.rs index ddcffc7c..3267c332 100644 --- a/crates/config/src/service/fs/model.rs +++ b/crates/config/src/service/fs/model.rs @@ -5,7 +5,7 @@ use std::{ use serde::{Deserialize, Serialize}; use serde_json::Value; -use spacegate_model::{constants::DEFAULT_API_PORT, ConfigItem, PluginInstanceId, PluginInstanceMap, PluginInstanceName, SgGateway, SgHttpRoute}; +use spacegate_model::{constants::DEFAULT_API_PORT, ConfigItem, ObservabilityConfig, PluginInstanceId, PluginInstanceMap, PluginInstanceName, SgGateway, SgHttpRoute}; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(untagged)] @@ -53,6 +53,7 @@ pub struct MainFileConfig

{ pub gateways: Vec>, pub plugins: PluginConfigs, pub api_port: u16, + pub observability: ObservabilityConfig, } impl

Default for MainFileConfig

{ @@ -61,6 +62,7 @@ impl

Default for MainFileConfig

{ gateways: Default::default(), plugins: Default::default(), api_port: DEFAULT_API_PORT, + observability: Default::default(), } } } @@ -140,6 +142,7 @@ impl MainFileConfig { gateways, plugins: self.plugins, api_port: self.api_port, + observability: self.observability, } } } @@ -191,6 +194,7 @@ impl MainFileConfig { gateways, plugins, api_port: Some(self.api_port), + observability: self.observability, } } } @@ -228,6 +232,7 @@ impl From for MainFileConfig { gateways, plugins, api_port: value.api_port.unwrap_or(DEFAULT_API_PORT), + observability: value.observability, } } } diff --git a/crates/config/src/service/k8s/convert/gateway_k8s_conv.rs b/crates/config/src/service/k8s/convert/gateway_k8s_conv.rs index f85188db..32019bd0 100644 --- a/crates/config/src/service/k8s/convert/gateway_k8s_conv.rs +++ b/crates/config/src/service/k8s/convert/gateway_k8s_conv.rs @@ -3,7 +3,7 @@ use std::{collections::BTreeMap, hash::Hasher}; use k8s_gateway_api::{Gateway, GatewaySpec, GatewayTlsConfig, Listener, SecretObjectReference}; use k8s_openapi::{api::core::v1::Secret, ByteString}; use kube::{api::ObjectMeta, ResourceExt}; -use spacegate_model::{ext::k8s::helper_struct::SgTargetKind, PluginInstanceId}; +use spacegate_model::{ext::k8s::helper_struct::SgTargetKind, ObservabilityConfig, PluginInstanceId}; use crate::{constants, ext::k8s::crd::sg_filter::K8sSgFilterSpecTargetRef, service::k8s::K8s, SgGateway, SgParameters}; @@ -81,7 +81,7 @@ impl SgGatewayConv for SgGateway { } } -pub(crate) trait SgParametersConv { +pub trait SgParametersConv { fn from_kube_gateway(gateway: &Gateway) -> Self; fn into_kube_gateway(self) -> BTreeMap; } @@ -107,18 +107,71 @@ impl SgParametersConv for SgParameters { if let Some(enable_x_request_id) = self.enable_x_request_id { ann.insert(crate::constants::GATEWAY_ANNOTATION_ENABLE_X_REQUEST_ID.to_string(), enable_x_request_id.to_string()); } + if self.observability.enabled { + ann.insert(crate::constants::GATEWAY_ANNOTATION_OTEL_ENABLED.to_string(), self.observability.enabled.to_string()); + ann.insert(crate::constants::GATEWAY_ANNOTATION_OTEL_SERVICE_NAME.to_string(), self.observability.service_name); + ann.insert(crate::constants::GATEWAY_ANNOTATION_OTEL_ENDPOINT.to_string(), self.observability.otlp_endpoint); + ann.insert(crate::constants::GATEWAY_ANNOTATION_OTEL_PROTOCOL.to_string(), self.observability.protocol.to_string()); + ann.insert( + crate::constants::GATEWAY_ANNOTATION_OTEL_TRACES_ENABLED.to_string(), + self.observability.traces.enabled.to_string(), + ); + ann.insert( + crate::constants::GATEWAY_ANNOTATION_OTEL_TRACES_SAMPLE_RATIO.to_string(), + self.observability.traces.sample_ratio.to_string(), + ); + ann.insert( + crate::constants::GATEWAY_ANNOTATION_OTEL_METRICS_ENABLED.to_string(), + self.observability.metrics.enabled.to_string(), + ); + ann.insert( + crate::constants::GATEWAY_ANNOTATION_OTEL_METRICS_EXPORT_INTERVAL_MS.to_string(), + self.observability.metrics.export_interval_ms.to_string(), + ); + ann.insert( + crate::constants::GATEWAY_ANNOTATION_OTEL_LOGS_ENABLED.to_string(), + self.observability.logs.enabled.to_string(), + ); + ann.insert(crate::constants::GATEWAY_ANNOTATION_OTEL_LOGS_LEVEL.to_string(), self.observability.logs.level); + } ann } fn from_kube_gateway(gateway: &Gateway) -> Self { let gateway_annotations = gateway.metadata.annotations.clone(); if let Some(gateway_annotations) = gateway_annotations { + let mut observability = ObservabilityConfig { + enabled: gateway_annotations.get(crate::constants::GATEWAY_ANNOTATION_OTEL_ENABLED).and_then(|v| v.parse::().ok()).unwrap_or_default(), + service_name: gateway_annotations + .get(crate::constants::GATEWAY_ANNOTATION_OTEL_SERVICE_NAME) + .cloned() + .unwrap_or_else(|| ObservabilityConfig::default().service_name), + otlp_endpoint: gateway_annotations.get(crate::constants::GATEWAY_ANNOTATION_OTEL_ENDPOINT).cloned().unwrap_or_else(|| ObservabilityConfig::default().otlp_endpoint), + protocol: gateway_annotations.get(crate::constants::GATEWAY_ANNOTATION_OTEL_PROTOCOL).and_then(|v| v.parse().ok()).unwrap_or_default(), + ..Default::default() + }; + observability.traces.enabled = + gateway_annotations.get(crate::constants::GATEWAY_ANNOTATION_OTEL_TRACES_ENABLED).and_then(|v| v.parse::().ok()).unwrap_or_default(); + observability.traces.sample_ratio = gateway_annotations + .get(crate::constants::GATEWAY_ANNOTATION_OTEL_TRACES_SAMPLE_RATIO) + .and_then(|v| v.parse::().ok()) + .unwrap_or_else(|| ObservabilityConfig::default().traces.sample_ratio); + observability.metrics.enabled = + gateway_annotations.get(crate::constants::GATEWAY_ANNOTATION_OTEL_METRICS_ENABLED).and_then(|v| v.parse::().ok()).unwrap_or_default(); + observability.metrics.export_interval_ms = gateway_annotations + .get(crate::constants::GATEWAY_ANNOTATION_OTEL_METRICS_EXPORT_INTERVAL_MS) + .and_then(|v| v.parse::().ok()) + .unwrap_or_else(|| ObservabilityConfig::default().metrics.export_interval_ms); + observability.logs.enabled = gateway_annotations.get(crate::constants::GATEWAY_ANNOTATION_OTEL_LOGS_ENABLED).and_then(|v| v.parse::().ok()).unwrap_or_default(); + observability.logs.level = + gateway_annotations.get(crate::constants::GATEWAY_ANNOTATION_OTEL_LOGS_LEVEL).cloned().unwrap_or_else(|| ObservabilityConfig::default().logs.level); SgParameters { redis_url: gateway_annotations.get(crate::constants::GATEWAY_ANNOTATION_REDIS_URL).map(|v| v.to_string()), log_level: gateway_annotations.get(crate::constants::GATEWAY_ANNOTATION_LOG_LEVEL).map(|v| v.to_string()), lang: gateway_annotations.get(crate::constants::GATEWAY_ANNOTATION_LANGUAGE).map(|v| v.to_string()), ignore_tls_verification: gateway_annotations.get(crate::constants::GATEWAY_ANNOTATION_IGNORE_TLS_VERIFICATION).and_then(|v| v.parse::().ok()), enable_x_request_id: gateway_annotations.get(crate::constants::GATEWAY_ANNOTATION_ENABLE_X_REQUEST_ID).and_then(|v| v.parse::().ok()), + observability, } } else { SgParameters { @@ -127,6 +180,7 @@ impl SgParametersConv for SgParameters { lang: None, ignore_tls_verification: None, enable_x_request_id: None, + observability: Default::default(), } } } diff --git a/crates/config/tests/test_k8s_config.rs b/crates/config/tests/test_k8s_config.rs index e9ab46b3..4c28d5d9 100644 --- a/crates/config/tests/test_k8s_config.rs +++ b/crates/config/tests/test_k8s_config.rs @@ -1,2 +1,58 @@ +#[cfg(feature = "k8s")] #[test] -fn test_k8s_config() {} +fn observability_annotations_roundtrip() { + use k8s_gateway_api::{Gateway, GatewaySpec}; + use kube::api::ObjectMeta; + use spacegate_config::service::k8s::convert::gateway_k8s_conv::SgParametersConv; + use spacegate_model::{ObservabilityConfig, OtlpProtocol, SgParameters}; + + let params = SgParameters { + observability: ObservabilityConfig { + enabled: true, + service_name: "spacegate-k8s".to_string(), + otlp_endpoint: "http://otel-collector:4317".to_string(), + protocol: OtlpProtocol::Grpc, + traces: spacegate_model::TraceConfig { + enabled: true, + sample_ratio: 0.25, + }, + metrics: spacegate_model::MetricConfig { + enabled: true, + export_interval_ms: 15000, + }, + logs: spacegate_model::LogConfig { + enabled: true, + level: "info".to_string(), + }, + ..Default::default() + }, + ..Default::default() + }; + + let annotations = params.into_kube_gateway(); + let gateway = Gateway { + metadata: ObjectMeta { + annotations: Some(annotations), + ..Default::default() + }, + spec: GatewaySpec { + gateway_class_name: Default::default(), + listeners: Default::default(), + addresses: Default::default(), + }, + status: Default::default(), + }; + + let parsed = SgParameters::from_kube_gateway(&gateway); + + assert!(parsed.observability.enabled); + assert_eq!(parsed.observability.service_name, "spacegate-k8s"); + assert_eq!(parsed.observability.otlp_endpoint, "http://otel-collector:4317"); + assert_eq!(parsed.observability.protocol, OtlpProtocol::Grpc); + assert!(parsed.observability.traces.enabled); + assert_eq!(parsed.observability.traces.sample_ratio, 0.25); + assert!(parsed.observability.metrics.enabled); + assert_eq!(parsed.observability.metrics.export_interval_ms, 15000); + assert!(parsed.observability.logs.enabled); + assert_eq!(parsed.observability.logs.level, "info"); +} diff --git a/crates/kernel/Cargo.toml b/crates/kernel/Cargo.toml index 770be75c..8d76d61a 100644 --- a/crates/kernel/Cargo.toml +++ b/crates/kernel/Cargo.toml @@ -36,6 +36,9 @@ mime_guess = "2" # log tracing = { workspace = true } +tracing-opentelemetry = { workspace = true } +opentelemetry = { workspace = true } +serde_json = { workspace = true } # runtime tokio = { workspace = true, features = ["net", "time", "macros", "fs"] } diff --git a/crates/kernel/src/extension.rs b/crates/kernel/src/extension.rs index 8b80d972..b0656d23 100644 --- a/crates/kernel/src/extension.rs +++ b/crates/kernel/src/extension.rs @@ -2,6 +2,8 @@ mod reflect; pub use reflect::*; mod gateway_name; pub use gateway_name::*; +mod route_name; +pub use route_name::*; mod matched; pub use matched::*; mod peer_addr; diff --git a/crates/kernel/src/extension/route_name.rs b/crates/kernel/src/extension/route_name.rs new file mode 100644 index 00000000..237f2974 --- /dev/null +++ b/crates/kernel/src/extension/route_name.rs @@ -0,0 +1,18 @@ +use std::{ops::Deref, sync::Arc}; + +#[derive(Debug, Clone)] +pub struct RouteName(pub Arc); + +impl RouteName { + pub fn new(name: impl Into>) -> Self { + Self(name.into()) + } +} + +impl Deref for RouteName { + type Target = str; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/crates/kernel/src/lib.rs b/crates/kernel/src/lib.rs index 7045cdff..cd8ec912 100644 --- a/crates/kernel/src/lib.rs +++ b/crates/kernel/src/lib.rs @@ -25,6 +25,8 @@ pub mod helper_layers; pub mod injector; /// tcp listener pub mod listener; +/// OpenTelemetry helpers. +pub mod observability; /// gateway service pub mod service; /// util functions and structs diff --git a/crates/kernel/src/observability.rs b/crates/kernel/src/observability.rs new file mode 100644 index 00000000..903b398c --- /dev/null +++ b/crates/kernel/src/observability.rs @@ -0,0 +1,503 @@ +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex, OnceLock}; +use std::time::Duration; + +use hyper::{header, Request, Response, StatusCode, Version}; +use opentelemetry::{global, KeyValue}; + +use crate::{extension::GatewayName, SgBody}; + +#[derive(Debug, Clone, Default)] +pub struct TelemetryContext { + fields: Arc>>, +} + +pub const MAX_TELEMETRY_KEY_LEN: usize = 128; +pub const MAX_TELEMETRY_VALUE_LEN: usize = 4096; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TelemetryError { + EmptyKey, + MissingNamespace, + ReservedPrefix, + InvalidKey, + KeyTooLong, + ValueTooLong, +} + +pub fn validate_telemetry_key(key: &str) -> Result<(), TelemetryError> { + if key.is_empty() { + return Err(TelemetryError::EmptyKey); + } + if key.len() > MAX_TELEMETRY_KEY_LEN { + return Err(TelemetryError::KeyTooLong); + } + if !key.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) { + return Err(TelemetryError::InvalidKey); + } + if !key.contains('.') { + return Err(TelemetryError::MissingNamespace); + } + if ["http.", "net.", "gateway.", "spacegate.", "otel."].iter().any(|prefix| key.starts_with(prefix)) { + return Err(TelemetryError::ReservedPrefix); + } + Ok(()) +} + +pub fn validate_telemetry_value(value: &str) -> Result<(), TelemetryError> { + if value.len() > MAX_TELEMETRY_VALUE_LEN { + return Err(TelemetryError::ValueTooLong); + } + Ok(()) +} + +impl TelemetryContext { + pub fn insert(&self, key: impl Into, value: impl Into) { + let Ok(mut fields) = self.fields.lock() else { + return; + }; + fields.insert(key.into(), value.into()); + } + + pub fn insert_checked(&self, key: impl Into, value: impl ToString) -> Result<(), TelemetryError> { + let key = key.into(); + let value = value.to_string(); + validate_telemetry_key(&key)?; + validate_telemetry_value(&value)?; + let Ok(mut fields) = self.fields.lock() else { + return Ok(()); + }; + fields.insert(key, value); + Ok(()) + } + + pub fn insert_namespaced(&self, namespace: &str, key: &str, value: impl ToString) -> Result<(), TelemetryError> { + self.insert_checked(format!("{namespace}.{key}"), value) + } + + pub fn snapshot(&self) -> BTreeMap { + self.fields.lock().map(|fields| fields.clone()).unwrap_or_default() + } + + pub fn is_empty(&self) -> bool { + self.fields.lock().map(|fields| fields.is_empty()).unwrap_or(true) + } +} + +#[derive(Debug, Clone)] +pub struct HttpMetricLabels { + pub gateway: String, + pub method: String, + pub status_code: String, + pub protocol_name: String, + pub protocol_version: String, + pub request_body_size: Option, + pub response_body_size: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AccessLogFields { + pub gateway: String, + pub method: String, + pub path: String, + pub host: String, + pub client_ip: String, + pub x_forwarded_for: String, + pub user_agent: String, + pub authority: String, + pub downstream_remote_address: String, + pub route_name: String, + pub upstream_host: String, + pub trace_id: String, + pub protocol_name: String, + pub protocol_version: String, + pub status_code: u16, + pub request_id: String, + pub peer_addr: String, + pub duration_ms: u64, + pub request_body_size: Option, + pub response_body_size: Option, + pub telemetry: BTreeMap, +} + +pub fn http_metric_labels(req: &Request, resp: &Response) -> HttpMetricLabels { + HttpMetricLabels { + gateway: req.extensions().get::().map(|g| g.to_string()).unwrap_or_else(|| "unknown".to_string()), + method: req.method().as_str().to_string(), + status_code: resp.status().as_u16().to_string(), + protocol_name: "http".to_string(), + protocol_version: http_protocol_version(req.version()), + request_body_size: content_length(req.headers()), + response_body_size: content_length(resp.headers()), + } +} + +pub fn access_log_fields( + gateway: impl Into, + method: impl Into, + path: impl Into, + host: impl Into, + client_ip: impl Into, + x_forwarded_for: impl Into, + user_agent: impl Into, + authority: impl Into, + downstream_remote_address: impl Into, + route_name: impl Into, + upstream_host: impl Into, + trace_id: impl Into, + protocol_version: impl Into, + status_code: StatusCode, + request_id: impl Into, + peer_addr: impl Into, + duration: Duration, + request_body_size: Option, + response_body_size: Option, + telemetry: BTreeMap, +) -> AccessLogFields { + AccessLogFields { + gateway: gateway.into(), + method: method.into(), + path: path.into(), + host: host.into(), + client_ip: client_ip.into(), + x_forwarded_for: x_forwarded_for.into(), + user_agent: user_agent.into(), + authority: authority.into(), + downstream_remote_address: downstream_remote_address.into(), + route_name: route_name.into(), + upstream_host: upstream_host.into(), + trace_id: trace_id.into(), + protocol_name: "http".to_string(), + protocol_version: protocol_version.into(), + status_code: status_code.as_u16(), + request_id: request_id.into(), + peer_addr: peer_addr.into(), + duration_ms: duration.as_millis() as u64, + request_body_size, + response_body_size, + telemetry, + } +} + +pub fn telemetry_json(fields: &BTreeMap) -> String { + serde_json::to_string(fields).unwrap_or_else(|_| "{}".to_string()) +} + +pub fn content_length(headers: &hyper::HeaderMap) -> Option { + headers.get(header::CONTENT_LENGTH)?.to_str().ok()?.parse().ok() +} + +pub fn header_value(headers: &hyper::HeaderMap, name: impl AsRef) -> String { + headers.get(name.as_ref()).and_then(|v| v.to_str().ok()).unwrap_or_default().to_string() +} + +pub fn first_x_forwarded_for(headers: &hyper::HeaderMap) -> Option { + header_value(headers, "x-forwarded-for").split(',').map(str::trim).find(|value| !value.is_empty()).map(str::to_string) +} + +pub fn client_ip(headers: &hyper::HeaderMap, peer_addr: std::net::SocketAddr) -> String { + first_x_forwarded_for(headers).unwrap_or_else(|| peer_addr.ip().to_string()) +} + +pub fn record_http_server_metrics(req: &Request, resp: &Response, duration: Duration) { + let labels = http_metric_labels(req, resp); + record_http_server_metrics_with_labels(labels, duration, resp.status().is_server_error() || resp.status().is_client_error()); +} + +pub fn record_http_server_metrics_with_labels(labels: HttpMetricLabels, duration: Duration, is_error: bool) { + let error_class = status_error_class_from_code(&labels.status_code); + let attrs = [ + KeyValue::new("gateway", labels.gateway), + KeyValue::new("http.request.method", labels.method), + KeyValue::new("http.response.status_code", labels.status_code), + KeyValue::new("network.protocol.name", labels.protocol_name), + KeyValue::new("network.protocol.version", labels.protocol_version), + ]; + let instruments = http_instruments(); + instruments.requests.add(1, &attrs); + instruments.duration.record(duration.as_secs_f64(), &attrs); + if let Some(size) = labels.request_body_size { + instruments.request_body_size.record(size, &attrs); + } + if let Some(size) = labels.response_body_size { + instruments.response_body_size.record(size, &attrs); + } + if is_error { + instruments.errors.add(1, &attrs); + } + match error_class { + Some(HttpErrorClass::Client) => instruments.errors_4xx.add(1, &attrs), + Some(HttpErrorClass::Server) => instruments.errors_5xx.add(1, &attrs), + None => {} + } +} + +pub fn record_http_server_active_request(labels: HttpMetricLabels, delta: i64) { + let attrs = [ + KeyValue::new("gateway", labels.gateway), + KeyValue::new("http.request.method", labels.method), + KeyValue::new("network.protocol.name", labels.protocol_name), + KeyValue::new("network.protocol.version", labels.protocol_version), + ]; + http_instruments().active_requests.add(delta, &attrs); +} + +#[derive(Debug)] +struct HttpInstruments { + requests: opentelemetry::metrics::Counter, + errors: opentelemetry::metrics::Counter, + errors_4xx: opentelemetry::metrics::Counter, + errors_5xx: opentelemetry::metrics::Counter, + active_requests: opentelemetry::metrics::UpDownCounter, + duration: opentelemetry::metrics::Histogram, + request_body_size: opentelemetry::metrics::Histogram, + response_body_size: opentelemetry::metrics::Histogram, +} + +fn http_instruments() -> &'static HttpInstruments { + static INSTRUMENTS: OnceLock = OnceLock::new(); + INSTRUMENTS.get_or_init(|| { + let meter = global::meter("spacegate_kernel"); + HttpInstruments { + requests: meter.u64_counter("http.server.requests").build(), + errors: meter.u64_counter("http.server.errors").build(), + errors_4xx: meter.u64_counter("http.server.errors.4xx").build(), + errors_5xx: meter.u64_counter("http.server.errors.5xx").build(), + active_requests: meter.i64_up_down_counter("http.server.active_requests").with_unit("{request}").build(), + duration: meter.f64_histogram("http.server.request.duration").with_unit("s").build(), + request_body_size: meter.u64_histogram("http.server.request.body.size").with_unit("By").build(), + response_body_size: meter.u64_histogram("http.server.response.body.size").with_unit("By").build(), + } + }) +} + +pub fn http_protocol_version(version: Version) -> String { + match version { + Version::HTTP_10 => "1.0", + Version::HTTP_11 => "1.1", + Version::HTTP_2 => "2", + Version::HTTP_3 => "3", + _ => "unknown", + } + .to_string() +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HttpErrorClass { + Client, + Server, +} + +pub fn status_error_class(status: StatusCode) -> Option { + if status.is_client_error() { + Some(HttpErrorClass::Client) + } else if status.is_server_error() { + Some(HttpErrorClass::Server) + } else { + None + } +} + +fn status_error_class_from_code(status_code: &str) -> Option { + StatusCode::from_u16(status_code.parse().ok()?).ok().and_then(status_error_class) +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, time::Duration}; + + use hyper::{header, Request, Response, StatusCode}; + + use crate::{extension::GatewayName, observability::http_metric_labels, SgBody}; + + #[test] + fn http_metric_labels_do_not_include_path() { + let mut req = Request::builder().method("GET").uri("/users/123?token=secret").body(SgBody::empty()).expect("request"); + req.extensions_mut().insert(GatewayName::new("gw-a")); + let resp = Response::builder().status(StatusCode::OK).body(SgBody::empty()).expect("response"); + + let labels = http_metric_labels(&req, &resp); + + assert_eq!(labels.gateway, "gw-a"); + assert_eq!(labels.method, "GET"); + assert_eq!(labels.status_code, "200"); + assert!(!format!("{labels:?}").contains("/users/123")); + } + + #[test] + fn http_metric_labels_use_protocol_name_and_version() { + let req = Request::builder().version(hyper::Version::HTTP_2).body(SgBody::empty()).expect("request"); + let resp = Response::builder().status(StatusCode::OK).body(SgBody::empty()).expect("response"); + + let labels = http_metric_labels(&req, &resp); + + assert_eq!(labels.protocol_name, "http"); + assert_eq!(labels.protocol_version, "2"); + } + + #[test] + fn http_protocol_version_maps_known_versions() { + assert_eq!(super::http_protocol_version(hyper::Version::HTTP_10), "1.0"); + assert_eq!(super::http_protocol_version(hyper::Version::HTTP_11), "1.1"); + assert_eq!(super::http_protocol_version(hyper::Version::HTTP_2), "2"); + assert_eq!(super::http_protocol_version(hyper::Version::HTTP_3), "3"); + } + + #[test] + fn status_error_classifies_4xx_and_5xx_only() { + assert_eq!(super::status_error_class(StatusCode::OK), None); + assert_eq!(super::status_error_class(StatusCode::BAD_REQUEST), Some(super::HttpErrorClass::Client)); + assert_eq!(super::status_error_class(StatusCode::INTERNAL_SERVER_ERROR), Some(super::HttpErrorClass::Server)); + } + + #[test] + fn access_log_fields_include_stable_request_data_and_telemetry() { + let telemetry = BTreeMap::from([("ai.asset_id".to_string(), "deepseek-chat".to_string()), ("ai.total_tokens".to_string(), "37".to_string())]); + + let fields = super::access_log_fields( + "gw-a", + "POST", + "/api/v1/model/deepseek-chat", + "example.local", + "203.0.113.10", + "203.0.113.10, 10.0.0.1", + "curl/8.7.1", + "example.local", + "127.0.0.1:12345", + "model-route", + "model.default.svc.cluster.local", + "4bf92f3577b34da6a3ce929d0e0e4736", + "1.1", + StatusCode::OK, + "req-1", + "127.0.0.1:12345", + Duration::from_millis(42), + Some(12), + Some(34), + telemetry, + ); + + assert_eq!(fields.gateway, "gw-a"); + assert_eq!(fields.method, "POST"); + assert_eq!(fields.client_ip, "203.0.113.10"); + assert_eq!(fields.x_forwarded_for, "203.0.113.10, 10.0.0.1"); + assert_eq!(fields.user_agent, "curl/8.7.1"); + assert_eq!(fields.authority, "example.local"); + assert_eq!(fields.downstream_remote_address, "127.0.0.1:12345"); + assert_eq!(fields.route_name, "model-route"); + assert_eq!(fields.upstream_host, "model.default.svc.cluster.local"); + assert_eq!(fields.trace_id, "4bf92f3577b34da6a3ce929d0e0e4736"); + assert_eq!(fields.status_code, 200); + assert_eq!(fields.duration_ms, 42); + assert_eq!(fields.request_body_size, Some(12)); + assert_eq!(fields.response_body_size, Some(34)); + assert_eq!(fields.telemetry.get("ai.asset_id").map(String::as_str), Some("deepseek-chat")); + } + + #[test] + fn http_metric_labels_include_body_sizes_from_content_length() { + let req = Request::builder().method("POST").header(header::CONTENT_LENGTH, "123").body(SgBody::empty()).expect("request"); + let resp = Response::builder().status(StatusCode::OK).header(header::CONTENT_LENGTH, "456").body(SgBody::empty()).expect("response"); + + let labels = http_metric_labels(&req, &resp); + + assert_eq!(labels.request_body_size, Some(123)); + assert_eq!(labels.response_body_size, Some(456)); + } + + #[test] + fn http_metric_labels_ignore_invalid_body_sizes() { + let req = Request::builder().header(header::CONTENT_LENGTH, "chunked").body(SgBody::empty()).expect("request"); + let resp = Response::builder().header(header::CONTENT_LENGTH, "-1").body(SgBody::empty()).expect("response"); + + let labels = http_metric_labels(&req, &resp); + + assert_eq!(labels.request_body_size, None); + assert_eq!(labels.response_body_size, None); + } + + #[test] + fn client_ip_prefers_first_x_forwarded_for_value() { + let req = Request::builder().header("x-forwarded-for", "203.0.113.10, 10.0.0.1").body(SgBody::empty()).expect("request"); + let peer = "127.0.0.1:12345".parse().expect("peer"); + + assert_eq!(super::client_ip(req.headers(), peer), "203.0.113.10"); + } + + #[test] + fn client_ip_falls_back_to_peer_ip() { + let req = Request::builder().body(SgBody::empty()).expect("request"); + let peer = "127.0.0.1:12345".parse().expect("peer"); + + assert_eq!(super::client_ip(req.headers(), peer), "127.0.0.1"); + } + + #[test] + fn telemetry_context_collects_plugin_fields() { + let context = super::TelemetryContext::default(); + + context.insert("ai.asset_id", "deepseek-chat"); + context.insert("ai.total_tokens", "37"); + + let fields = context.snapshot(); + assert_eq!(fields.get("ai.asset_id").map(String::as_str), Some("deepseek-chat")); + assert_eq!(fields.get("ai.total_tokens").map(String::as_str), Some("37")); + } + + #[test] + fn telemetry_key_validation_accepts_namespaced_keys() { + assert!(super::validate_telemetry_key("ai.total_tokens").is_ok()); + assert!(super::validate_telemetry_key("mcp.tool-name").is_ok()); + assert!(super::validate_telemetry_key("auth.api_key_hash").is_ok()); + } + + #[test] + fn telemetry_key_validation_rejects_bad_keys() { + assert_eq!(super::validate_telemetry_key(""), Err(super::TelemetryError::EmptyKey)); + assert_eq!(super::validate_telemetry_key("total_tokens"), Err(super::TelemetryError::MissingNamespace)); + assert_eq!(super::validate_telemetry_key("ai total_tokens"), Err(super::TelemetryError::InvalidKey)); + assert_eq!(super::validate_telemetry_key("http.status_code"), Err(super::TelemetryError::ReservedPrefix)); + assert_eq!(super::validate_telemetry_key("spacegate.internal"), Err(super::TelemetryError::ReservedPrefix)); + } + + #[test] + fn telemetry_value_validation_rejects_oversized_values() { + let value = "x".repeat(super::MAX_TELEMETRY_VALUE_LEN + 1); + assert_eq!(super::validate_telemetry_value(&value), Err(super::TelemetryError::ValueTooLong)); + } + + #[test] + fn telemetry_context_checked_insert_rejects_invalid_key_without_mutating_context() { + let context = super::TelemetryContext::default(); + + let result = context.insert_checked("total_tokens", "37"); + + assert_eq!(result, Err(super::TelemetryError::MissingNamespace)); + assert!(context.snapshot().is_empty()); + } + + #[test] + fn telemetry_context_namespaced_insert_builds_stable_key() { + let context = super::TelemetryContext::default(); + + context.insert_namespaced("ai", "total_tokens", 37).expect("insert"); + + let fields = context.snapshot(); + assert_eq!(fields.get("ai.total_tokens").map(String::as_str), Some("37")); + } + + #[test] + fn telemetry_json_serializes_plugin_defined_fields() { + let fields = BTreeMap::from([ + ("ai.asset_id".to_string(), "deepseek-chat".to_string()), + ("ai.total_tokens".to_string(), "37".to_string()), + ("mcp.tool".to_string(), "search".to_string()), + ]); + + let json = super::telemetry_json(&fields); + + assert!(json.contains("\"ai.asset_id\":\"deepseek-chat\"")); + assert!(json.contains("\"ai.total_tokens\":\"37\"")); + assert!(json.contains("\"mcp.tool\":\"search\"")); + } +} diff --git a/crates/kernel/src/service.rs b/crates/kernel/src/service.rs index 3cc1365d..8100be88 100644 --- a/crates/kernel/src/service.rs +++ b/crates/kernel/src/service.rs @@ -3,11 +3,18 @@ use std::{convert::Infallible, net::SocketAddr, sync::Arc}; use futures_util::future::BoxFuture; use hyper::{body::Incoming, Request, Response}; use hyper_util::rt::TokioIo; +use opentelemetry::trace::TraceContextExt; use tokio::net::TcpStream; use tokio_rustls::rustls; +use tracing::Instrument; +use tracing_opentelemetry::OpenTelemetrySpanExt; use crate::{ - extension::{EnterTime, PeerAddr, Reflect}, + extension::{BackendHost, EnterTime, PeerAddr, Reflect, RouteName}, + observability::{ + access_log_fields, client_ip, content_length, header_value, http_protocol_version, record_http_server_active_request, record_http_server_metrics_with_labels, + telemetry_json, HttpMetricLabels, TelemetryContext, + }, ArcHyperService, BoxResult, SgBody, }; @@ -26,13 +33,19 @@ type ConnectionBuilder = hyper_util::server::conn::auto::Builder, connection_builder: ConnectionBuilder, } impl Http { pub fn new(service: ArcHyperService) -> Self { + Self::with_gateway_name(service, Arc::::from("unknown")) + } + + pub fn with_gateway_name(service: ArcHyperService, gateway_name: Arc) -> Self { Self { inner_service: service, + gateway_name, connection_builder: ConnectionBuilder::new(Default::default()), } } @@ -59,7 +72,7 @@ impl TcpService for Http { } fn handle(&self, stream: TcpStream, peer: SocketAddr) -> BoxFuture<'static, BoxResult<()>> { let io = TokioIo::new(stream); - let service = HyperServiceAdapter::new(self.inner_service.clone(), peer); + let service = HyperServiceAdapter::with_gateway_name(self.inner_service.clone(), peer, self.gateway_name.clone()); let builder = self.connection_builder.clone(); Box::pin(async move { let conn = builder.serve_connection_with_upgrades(io, service); @@ -70,14 +83,20 @@ impl TcpService for Http { #[derive(Debug)] pub struct Https { inner_service: ArcHyperService, + gateway_name: Arc, tls_config: Arc, connection_builder: ConnectionBuilder, } impl Https { pub fn new(service: ArcHyperService, tls_config: rustls::ServerConfig) -> Self { + Self::with_gateway_name(service, tls_config, Arc::::from("unknown")) + } + + pub fn with_gateway_name(service: ArcHyperService, tls_config: rustls::ServerConfig, gateway_name: Arc) -> Self { Self { inner_service: service, + gateway_name, tls_config: Arc::new(tls_config), connection_builder: ConnectionBuilder::new(Default::default()), } @@ -95,7 +114,7 @@ impl TcpService for Https { peeked.starts_with(b"\x16\x03") } fn handle(&self, stream: TcpStream, peer: SocketAddr) -> BoxFuture<'static, BoxResult<()>> { - let service = HyperServiceAdapter::new(self.inner_service.clone(), peer); + let service = HyperServiceAdapter::with_gateway_name(self.inner_service.clone(), peer, self.gateway_name.clone()); let builder = self.connection_builder.clone(); let connector = tokio_rustls::TlsAcceptor::from(self.tls_config.clone()); Box::pin(async move { @@ -114,6 +133,7 @@ where { service: S, peer: SocketAddr, + gateway_name: Arc, } impl HyperServiceAdapter @@ -122,7 +142,15 @@ where S::Future: Send + 'static, { pub fn new(service: S, peer: SocketAddr) -> Self { - Self { service, peer } + Self::with_gateway_name(service, peer, Arc::::from("unknown")) + } + + pub fn with_gateway_name(service: S, peer: SocketAddr, gateway_name: Arc) -> Self { + Self { service, peer, gateway_name } + } + + pub fn gateway_name(&self) -> &str { + self.gateway_name.as_ref() } } @@ -147,28 +175,141 @@ where let enter_time = EnterTime::new(); let service = self.service.clone(); let mut req = req.map(SgBody::new); + let method = req.method().clone(); + let method_label = method.as_str().to_string(); + let path = req.uri().path().to_string(); + let host = req.uri().host().map(str::to_string).or_else(|| req.headers().get(hyper::header::HOST).and_then(|v| v.to_str().ok()).map(str::to_string)).unwrap_or_default(); + let protocol = format!("{:?}", req.version()); + let protocol_version_label = http_protocol_version(req.version()); + let request_id = req.headers().get("x-request-id").and_then(|v| v.to_str().ok()).unwrap_or_default().to_string(); + let x_forwarded_for = header_value(req.headers(), "x-forwarded-for"); + let user_agent = header_value(req.headers(), "user-agent"); + let client_ip_label = client_ip(req.headers(), self.peer); + let request_body_size = content_length(req.headers()); + let peer_addr_label = self.peer.to_string(); + let span = tracing::info_span!( + "http.server.request", + http.method = %method, + http.path = %path, + http.host = %host, + http.protocol = %protocol, + http.status_code = tracing::field::Empty, + request_id = %request_id, + peer_addr = %self.peer, + duration_ms = tracing::field::Empty + ); + let gateway_label = self.gateway_name.to_string(); + let telemetry_context = TelemetryContext::default(); + let active_request_labels = HttpMetricLabels { + gateway: gateway_label.clone(), + method: method_label.clone(), + status_code: "active".to_string(), + protocol_name: "http".to_string(), + protocol_version: protocol_version_label.clone(), + request_body_size, + response_body_size: None, + }; + record_http_server_active_request(active_request_labels.clone(), 1); let mut reflect = Reflect::default(); // let method = req.method().clone(); reflect.insert(enter_time); req.extensions_mut().insert(reflect); req.extensions_mut().insert(PeerAddr(self.peer)); req.extensions_mut().insert(enter_time); - Box::pin(async move { - let resp = service.call(req).await.expect("infallible"); - // if method != hyper::Method::HEAD && method != hyper::Method::OPTIONS && method != hyper::Method::CONNECT { - // with_length_or_chunked(&mut resp); - // } - let status = resp.status(); - if status.is_server_error() { - tracing::warn!(status = ?status, headers = ?resp.headers(), "server error response"); - } else if status.is_client_error() { - tracing::debug!(status = ?status, headers = ?resp.headers(), "client error response"); - } else if status.is_success() { - tracing::trace!(status = ?status, headers = ?resp.headers(), "success response"); + req.extensions_mut().insert(telemetry_context.clone()); + let span_for_recording = span.clone(); + Box::pin( + async move { + let resp = service.call(req).await.expect("infallible"); + // if method != hyper::Method::HEAD && method != hyper::Method::OPTIONS && method != hyper::Method::CONNECT { + // with_length_or_chunked(&mut resp); + // } + let status = resp.status(); + if status.is_server_error() { + tracing::warn!(status = ?status, headers = ?resp.headers(), "server error response"); + } else if status.is_client_error() { + tracing::debug!(status = ?status, headers = ?resp.headers(), "client error response"); + } else if status.is_success() { + tracing::trace!(status = ?status, headers = ?resp.headers(), "success response"); + } + let latency = enter_time.elapsed(); + span_for_recording.record("http.status_code", status.as_u16()); + span_for_recording.record("duration_ms", latency.as_millis() as u64); + let response_body_size = content_length(resp.headers()); + let access_request_id = resp.headers().get("x-request-id").and_then(|v| v.to_str().ok()).map(str::to_string).unwrap_or(request_id); + tracing::trace!(latency = ?latency, "request finished"); + let authority = host.clone(); + let route_name = resp.extensions().get::().map(|route| route.to_string()).unwrap_or_default(); + let upstream_host = resp.extensions().get::().map(|host| host.to_string()).unwrap_or_default(); + let trace_id = span_for_recording.context().span().span_context().trace_id().to_string(); + record_http_server_metrics_with_labels( + HttpMetricLabels { + gateway: gateway_label.clone(), + method: method_label.clone(), + status_code: status.as_u16().to_string(), + protocol_name: "http".to_string(), + protocol_version: protocol_version_label.clone(), + request_body_size, + response_body_size, + }, + latency, + status.is_server_error() || status.is_client_error(), + ); + let access_log = access_log_fields( + gateway_label, + method_label, + path, + host, + client_ip_label, + x_forwarded_for, + user_agent, + authority, + peer_addr_label.clone(), + route_name, + upstream_host, + trace_id, + protocol_version_label, + status, + access_request_id, + peer_addr_label, + latency, + request_body_size, + response_body_size, + telemetry_context.snapshot(), + ); + let telemetry = telemetry_json(&access_log.telemetry); + tracing::info!( + event = "http_access", + gateway = %access_log.gateway, + method = %access_log.method, + path = %access_log.path, + host = %access_log.host, + authority = %access_log.authority, + client_ip = %access_log.client_ip, + x_forwarded_for = %access_log.x_forwarded_for, + user_agent = %access_log.user_agent, + downstream_remote_address = %access_log.downstream_remote_address, + route_name = %access_log.route_name, + upstream_host = %access_log.upstream_host, + trace_id = %access_log.trace_id, + protocol_name = %access_log.protocol_name, + protocol_version = %access_log.protocol_version, + status_code = access_log.status_code, + request_id = %access_log.request_id, + peer_addr = %access_log.peer_addr, + duration_ms = access_log.duration_ms, + bytes_received = ?access_log.request_body_size, + bytes_sent = ?access_log.response_body_size, + request_body_size = ?access_log.request_body_size, + response_body_size = ?access_log.response_body_size, + telemetry = %telemetry, + "http access log" + ); + record_http_server_active_request(active_request_labels, -1); + Ok(resp) } - tracing::trace!(latency = ?enter_time.elapsed(), "request finished"); - Ok(resp) - }) + .instrument(span), + ) } } @@ -179,4 +320,25 @@ impl ArcHyperService { pub fn https(self, tls_config: rustls::ServerConfig) -> Https { Https::new(self, tls_config) } + pub fn http_with_gateway_name(self, gateway_name: Arc) -> Http { + Http::with_gateway_name(self, gateway_name) + } + pub fn https_with_gateway_name(self, tls_config: rustls::ServerConfig, gateway_name: Arc) -> Https { + Https::with_gateway_name(self, tls_config, gateway_name) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hyper_service_adapter_keeps_gateway_name_from_listener() { + let service = hyper::service::service_fn(|_req: Request| async { Ok::<_, Infallible>(Response::new(SgBody::empty())) }); + let peer = "127.0.0.1:12345".parse().expect("peer"); + + let adapter = HyperServiceAdapter::with_gateway_name(service, peer, Arc::::from("gw-a")); + + assert_eq!(adapter.gateway_name(), "gw-a"); + } } diff --git a/crates/kernel/src/service/http_gateway.rs b/crates/kernel/src/service/http_gateway.rs index fc629cac..d1c9c8cd 100644 --- a/crates/kernel/src/service/http_gateway.rs +++ b/crates/kernel/src/service/http_gateway.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, ops::Index, sync::Arc}; use crate::{ backend_service::ArcHyperService, - extension::{GatewayName, MatchedSgRouter}, + extension::{GatewayName, MatchedSgRouter, Reflect, RouteName}, helper_layers::{ map_request::{add_extension::add_extension, MapRequestLayer}, reload::Reloader, @@ -96,6 +96,7 @@ impl Router for GatewayRouter { if let Some(ref matches) = matches { for m in matches.as_ref() { if m.match_request(req) { + insert_route_name(req, self.routers.as_ref()[*route_index].name.clone()); req.extensions_mut().insert(MatchedSgRouter(m.clone())); tracing::trace!("matches {m:?} [{route_index},{idx1}:{_p}]"); if let Err(e) = m.rewrite(req) { @@ -107,6 +108,7 @@ impl Router for GatewayRouter { } continue; } else { + insert_route_name(req, self.routers.as_ref()[*route_index].name.clone()); tracing::trace!("matches wildcard [{route_index},{idx1}:{_p}]"); return Some(index); } @@ -150,6 +152,7 @@ pub fn create_http_router<'a>(routes: impl Iterator, fallb } services.push(rules_services); routers.push(HttpRouter { + name: route.name.clone().into(), hostnames: route.hostnames.clone().into(), rules: rules_router.into_iter().map(|x| x.map(|v| v.into_iter().map(Arc::new).collect::>())).collect(), ext: route.ext.clone(), @@ -169,3 +172,11 @@ pub fn create_http_router<'a>(routes: impl Iterator, fallb fallback, ) } + +fn insert_route_name(req: &mut Request, name: Arc) { + let route_name = RouteName::new(name); + if let Some(reflect) = req.extensions_mut().get_mut::() { + reflect.insert(route_name.clone()); + } + req.extensions_mut().insert(route_name); +} diff --git a/crates/kernel/src/service/http_route.rs b/crates/kernel/src/service/http_route.rs index 3fac72fd..5f97d552 100644 --- a/crates/kernel/src/service/http_route.rs +++ b/crates/kernel/src/service/http_route.rs @@ -44,6 +44,7 @@ impl HttpRoute { } #[derive(Debug, Clone)] pub struct HttpRouter { + pub name: Arc, pub hostnames: Arc<[String]>, pub rules: Arc<[Option]>>]>, pub ext: hyper::http::Extensions, diff --git a/crates/model/src/constants.rs b/crates/model/src/constants.rs index c8c36a50..88e46005 100644 --- a/crates/model/src/constants.rs +++ b/crates/model/src/constants.rs @@ -8,6 +8,16 @@ pub const GATEWAY_ANNOTATION_LOG_LEVEL: &str = "log_level"; pub const GATEWAY_ANNOTATION_LANGUAGE: &str = "lang"; pub const GATEWAY_ANNOTATION_IGNORE_TLS_VERIFICATION: &str = "ignore_tls_verification"; pub const GATEWAY_ANNOTATION_ENABLE_X_REQUEST_ID: &str = "enable_x_request_id"; +pub const GATEWAY_ANNOTATION_OTEL_ENABLED: &str = "spacegate.io/otel-enabled"; +pub const GATEWAY_ANNOTATION_OTEL_SERVICE_NAME: &str = "spacegate.io/otel-service-name"; +pub const GATEWAY_ANNOTATION_OTEL_ENDPOINT: &str = "spacegate.io/otel-endpoint"; +pub const GATEWAY_ANNOTATION_OTEL_PROTOCOL: &str = "spacegate.io/otel-protocol"; +pub const GATEWAY_ANNOTATION_OTEL_TRACES_ENABLED: &str = "spacegate.io/otel-traces-enabled"; +pub const GATEWAY_ANNOTATION_OTEL_TRACES_SAMPLE_RATIO: &str = "spacegate.io/otel-traces-sample-ratio"; +pub const GATEWAY_ANNOTATION_OTEL_METRICS_ENABLED: &str = "spacegate.io/otel-metrics-enabled"; +pub const GATEWAY_ANNOTATION_OTEL_METRICS_EXPORT_INTERVAL_MS: &str = "spacegate.io/otel-metrics-export-interval-ms"; +pub const GATEWAY_ANNOTATION_OTEL_LOGS_ENABLED: &str = "spacegate.io/otel-logs-enabled"; +pub const GATEWAY_ANNOTATION_OTEL_LOGS_LEVEL: &str = "spacegate.io/otel-logs-level"; pub const SG_FILTER_KIND: &str = "sgfilter"; pub const DEFAULT_NAMESPACE: &str = "default"; diff --git a/crates/model/src/gateway.rs b/crates/model/src/gateway.rs index 22f39461..cc2b1d16 100644 --- a/crates/model/src/gateway.rs +++ b/crates/model/src/gateway.rs @@ -2,7 +2,7 @@ use std::{fmt::Display, net::IpAddr}; use serde::{Deserialize, Serialize}; -use super::plugin::PluginInstanceId; +use super::{observability::ObservabilityConfig, plugin::PluginInstanceId}; /// Gateway represents an instance of a service-traffic handling infrastructure /// by binding Listeners to a set of IP addresses. @@ -69,6 +69,7 @@ pub struct SgParameters { #[serde(skip_serializing_if = "Option::is_none")] /// Add request id for every request pub enable_x_request_id: Option, + pub observability: ObservabilityConfig, } /// Listener embodies the concept of a logical endpoint where a Gateway accepts network connections. diff --git a/crates/model/src/lib.rs b/crates/model/src/lib.rs index 89985a76..7d99c043 100644 --- a/crates/model/src/lib.rs +++ b/crates/model/src/lib.rs @@ -3,6 +3,9 @@ use std::{collections::BTreeMap, fmt::Debug}; pub use plugin::*; +pub mod observability; +pub use observability::*; + pub mod gateway; pub use gateway::*; @@ -62,6 +65,7 @@ pub struct Config { #[cfg_attr(feature = "typegen", ts(as = "crate::plugin::PluginInstanceMapTs"))] pub plugins: PluginInstanceMap, pub api_port: Option, + pub observability: ObservabilityConfig, } #[allow(clippy::derivable_impls)] @@ -74,6 +78,7 @@ impl Default for Config { api_port: Some(crate::constants::DEFAULT_API_PORT), #[cfg(not(feature = "ext-axum"))] api_port: None, + observability: Default::default(), } } } diff --git a/crates/model/src/observability.rs b/crates/model/src/observability.rs new file mode 100644 index 00000000..1b1ee38e --- /dev/null +++ b/crates/model/src/observability.rs @@ -0,0 +1,109 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +#[cfg_attr(feature = "typegen", derive(ts_rs::TS), ts(export))] +#[serde(default)] +pub struct ObservabilityConfig { + pub enabled: bool, + pub service_name: String, + pub otlp_endpoint: String, + pub protocol: OtlpProtocol, + pub traces: TraceConfig, + pub metrics: MetricConfig, + pub logs: LogConfig, +} + +impl Default for ObservabilityConfig { + fn default() -> Self { + Self { + enabled: false, + service_name: "spacegate".to_string(), + otlp_endpoint: "http://localhost:4317".to_string(), + protocol: OtlpProtocol::Grpc, + traces: TraceConfig::default(), + metrics: MetricConfig::default(), + logs: LogConfig::default(), + } + } +} + +#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Default)] +#[cfg_attr(feature = "typegen", derive(ts_rs::TS), ts(export))] +#[serde(rename_all = "lowercase")] +pub enum OtlpProtocol { + #[default] + Grpc, + Http, +} + +impl std::fmt::Display for OtlpProtocol { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + OtlpProtocol::Grpc => write!(f, "grpc"), + OtlpProtocol::Http => write!(f, "http"), + } + } +} + +impl std::str::FromStr for OtlpProtocol { + type Err = crate::BoxError; + + fn from_str(s: &str) -> Result { + match s { + "grpc" => Ok(OtlpProtocol::Grpc), + "http" | "http/protobuf" => Ok(OtlpProtocol::Http), + _ => Err(format!("invalid otlp protocol: {s}").into()), + } + } +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +#[cfg_attr(feature = "typegen", derive(ts_rs::TS), ts(export))] +#[serde(default)] +pub struct TraceConfig { + pub enabled: bool, + pub sample_ratio: f64, +} + +impl Default for TraceConfig { + fn default() -> Self { + Self { + enabled: false, + sample_ratio: 1.0, + } + } +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "typegen", derive(ts_rs::TS), ts(export))] +#[serde(default)] +pub struct MetricConfig { + pub enabled: bool, + pub export_interval_ms: u64, +} + +impl Default for MetricConfig { + fn default() -> Self { + Self { + enabled: false, + export_interval_ms: 60_000, + } + } +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "typegen", derive(ts_rs::TS), ts(export))] +#[serde(default)] +pub struct LogConfig { + pub enabled: bool, + pub level: String, +} + +impl Default for LogConfig { + fn default() -> Self { + Self { + enabled: false, + level: "info".to_string(), + } + } +} diff --git a/crates/model/tests/test_parse_config.rs b/crates/model/tests/test_parse_config.rs index 2fa76a3f..e5e3e0bd 100644 --- a/crates/model/tests/test_parse_config.rs +++ b/crates/model/tests/test_parse_config.rs @@ -19,3 +19,134 @@ fn test_parse_config() { } } } + +#[test] +fn observability_defaults_to_disabled() { + let config = Config::default(); + + assert!(!config.observability.enabled); + assert_eq!(config.observability.service_name, "spacegate"); + assert_eq!(config.observability.otlp_endpoint, "http://localhost:4317"); + assert!(!config.observability.traces.enabled); + assert!(!config.observability.metrics.enabled); + assert!(!config.observability.logs.enabled); +} + +#[test] +fn observability_can_be_parsed_from_config() { + let file = r#" +[observability] +enabled = true +service_name = "spacegate-test" +otlp_endpoint = "http://collector:4317" +protocol = "grpc" + +[observability.traces] +enabled = true +sample_ratio = 0.5 + +[observability.metrics] +enabled = true +export_interval_ms = 10000 + +[observability.logs] +enabled = true +level = "warn" +"#; + + let config = toml::from_str::(file).expect("parse config"); + + assert!(config.observability.enabled); + assert_eq!(config.observability.service_name, "spacegate-test"); + assert_eq!(config.observability.otlp_endpoint, "http://collector:4317"); + assert_eq!(config.observability.protocol, spacegate_model::OtlpProtocol::Grpc); + assert!(config.observability.traces.enabled); + assert_eq!(config.observability.traces.sample_ratio, 0.5); + assert!(config.observability.metrics.enabled); + assert_eq!(config.observability.metrics.export_interval_ms, 10000); + assert!(config.observability.logs.enabled); + assert_eq!(config.observability.logs.level, "warn"); +} + +#[test] +fn local_otel_json_config_shape_can_be_parsed() { + let file = r#" +{ + "api_port": 9876, + "observability": { + "enabled": true, + "service_name": "spacegate-local-otel", + "otlp_endpoint": "http://127.0.0.1:4317", + "protocol": "grpc", + "traces": { + "enabled": true, + "sample_ratio": 1.0 + }, + "metrics": { + "enabled": true, + "export_interval_ms": 5000 + }, + "logs": { + "enabled": true, + "level": "info" + } + }, + "gateways": { + "local": { + "gateway": { + "name": "local", + "parameters": { + "enable_x_request_id": true + }, + "listeners": [ + { + "name": "http", + "ip": "0.0.0.0", + "port": 9000, + "protocol": { + "type": "http" + } + } + ] + }, + "routes": { + "root": { + "route_name": "root", + "rules": [ + { + "matches": [ + { + "path": { + "kind": "Prefix", + "value": "/" + } + } + ], + "backends": [ + { + "host": { + "kind": "Host", + "host": "127.0.0.1" + }, + "port": 18080, + "protocol": "http", + "weight": 1 + } + ] + } + ] + } + } + } + } +} +"#; + + let config = serde_json::from_str::(file).expect("parse local otel json config"); + let gateway = config.gateways.get("local").expect("local gateway"); + + assert_eq!(gateway.gateway.name, "local"); + assert_eq!(gateway.gateway.listeners.len(), 1); + assert_eq!(gateway.gateway.listeners[0].port, 9000); + assert!(gateway.routes.contains_key("root")); +} diff --git a/crates/plugin-wasm/Cargo.toml b/crates/plugin-wasm/Cargo.toml index b8587fd3..be1cf7b3 100644 --- a/crates/plugin-wasm/Cargo.toml +++ b/crates/plugin-wasm/Cargo.toml @@ -24,6 +24,7 @@ serde_json = { workspace = true } serde_yaml = "0.9" tracing = { workspace = true } +opentelemetry = { workspace = true } thiserror = "1" once_cell = "1.19" sha2 = "0.10" diff --git a/crates/plugin-wasm/src/shared.rs b/crates/plugin-wasm/src/shared.rs index d30f23ba..c350a136 100644 --- a/crates/plugin-wasm/src/shared.rs +++ b/crates/plugin-wasm/src/shared.rs @@ -15,6 +15,7 @@ use std::collections::{HashMap, VecDeque}; use std::sync::{Mutex, RwLock}; use once_cell::sync::Lazy; +use opentelemetry::global; use crate::abi::MetricType; @@ -142,8 +143,14 @@ pub fn queue_dequeue(qid: u32) -> (QueueOpResult, Option>) { struct MetricEntry { kind: MetricType, value: u64, - #[allow(dead_code)] - name: String, + instrument: OtelMetricInstrument, +} + +#[derive(Debug)] +enum OtelMetricInstrument { + Counter(opentelemetry::metrics::Counter), + Gauge(opentelemetry::metrics::Gauge), + Histogram(opentelemetry::metrics::Histogram), } #[derive(Debug, Default)] @@ -172,14 +179,13 @@ pub fn metric_define(kind: MetricType, name: &str) -> u32 { } g.next_id = g.next_id.wrapping_add(1).max(1); let id = g.next_id; - g.by_id.insert( - id, - MetricEntry { - kind, - value: 0, - name: name.to_string(), - }, - ); + let meter = global::meter("spacegate_plugin_wasm"); + let instrument = match kind { + MetricType::Counter => OtelMetricInstrument::Counter(meter.u64_counter(name.to_string()).build()), + MetricType::Gauge => OtelMetricInstrument::Gauge(meter.i64_gauge(name.to_string()).build()), + MetricType::Histogram => OtelMetricInstrument::Histogram(meter.u64_histogram(name.to_string()).build()), + }; + g.by_id.insert(id, MetricEntry { kind, value: 0, instrument }); g.by_name.insert(name.to_string(), id); id } @@ -192,6 +198,11 @@ pub fn metric_record(id: u32, value: u64) -> MetricOpResult { match g.by_id.get_mut(&id) { Some(m) => { m.value = value; + match &m.instrument { + OtelMetricInstrument::Counter(counter) => counter.add(value, &[]), + OtelMetricInstrument::Gauge(gauge) => gauge.record(value as i64, &[]), + OtelMetricInstrument::Histogram(histogram) => histogram.record(value, &[]), + } MetricOpResult::Ok } None => MetricOpResult::NotFound, @@ -214,6 +225,11 @@ pub fn metric_increment(id: u32, delta: i64) -> MetricOpResult { } else { m.value = m.value.saturating_sub((-delta) as u64); } + match &m.instrument { + OtelMetricInstrument::Counter(counter) => counter.add(delta.max(0) as u64, &[]), + OtelMetricInstrument::Gauge(gauge) => gauge.record(m.value as i64, &[]), + OtelMetricInstrument::Histogram(histogram) => histogram.record(m.value, &[]), + } MetricOpResult::Ok } diff --git a/crates/plugin/src/lib.rs b/crates/plugin/src/lib.rs index 16849abe..186fa532 100644 --- a/crates/plugin/src/lib.rs +++ b/crates/plugin/src/lib.rs @@ -33,6 +33,21 @@ pub mod plugins; pub use schemars; pub use spacegate_model; pub use spacegate_model::{plugin_meta, PluginAttributes, PluginConfig, PluginInstanceId, PluginInstanceMap, PluginInstanceName, PluginMetaData}; + +pub fn set_telemetry_field(req: &SgRequest, key: impl Into, value: impl ToString) -> Result<(), spacegate_kernel::observability::TelemetryError> { + if let Some(context) = req.extensions().get::() { + context.insert_checked(key, value)?; + } + Ok(()) +} + +pub fn set_plugin_telemetry_field(req: &SgRequest, namespace: &str, key: &str, value: impl ToString) -> Result<(), spacegate_kernel::observability::TelemetryError> { + if let Some(context) = req.extensions().get::() { + context.insert_namespaced(namespace, key, value)?; + } + Ok(()) +} + /// # Plugin Trait /// It's a easy way to define a plugin through this trait. /// You should give a unique [`code`](Plugin::CODE) for the plugin, diff --git a/crates/plugin/src/plugins/limit.rs b/crates/plugin/src/plugins/limit.rs index e5711011..c52c4422 100644 --- a/crates/plugin/src/plugins/limit.rs +++ b/crates/plugin/src/plugins/limit.rs @@ -126,7 +126,7 @@ impl Plugin for RateLimitPlugin { if result == EXCEEDED { let mut response = Response::::with_code_message(StatusCode::TOO_MANY_REQUESTS, "[SG.Filter.Limit] too many requests"); - response.extensions_mut().insert(self.report( ip)); + response.extensions_mut().insert(self.report(ip)); return Ok(response); } Ok(inner.call(req).await) diff --git a/crates/plugin/tests/test_telemetry.rs b/crates/plugin/tests/test_telemetry.rs new file mode 100644 index 00000000..1a44bdaa --- /dev/null +++ b/crates/plugin/tests/test_telemetry.rs @@ -0,0 +1,38 @@ +use spacegate_plugin::{set_plugin_telemetry_field, set_telemetry_field, SgBody}; + +fn request_with_telemetry() -> hyper::Request { + let mut req = hyper::Request::builder().body(SgBody::empty()).expect("request"); + req.extensions_mut().insert(spacegate_kernel::observability::TelemetryContext::default()); + req +} + +#[test] +fn set_telemetry_field_writes_checked_request_context() { + let req = request_with_telemetry(); + + set_telemetry_field(&req, "ai.asset_id", "deepseek-chat").expect("insert"); + set_telemetry_field(&req, "ai.total_tokens", 37).expect("insert"); + + let fields = req.extensions().get::().expect("telemetry context").snapshot(); + assert_eq!(fields.get("ai.asset_id").map(String::as_str), Some("deepseek-chat")); + assert_eq!(fields.get("ai.total_tokens").map(String::as_str), Some("37")); +} + +#[test] +fn set_plugin_telemetry_field_adds_namespace() { + let req = request_with_telemetry(); + + set_plugin_telemetry_field(&req, "mcp", "tool", "search").expect("insert"); + + let fields = req.extensions().get::().expect("telemetry context").snapshot(); + assert_eq!(fields.get("mcp.tool").map(String::as_str), Some("search")); +} + +#[test] +fn set_telemetry_field_rejects_unqualified_key() { + let req = request_with_telemetry(); + + let result = set_telemetry_field(&req, "total_tokens", 37); + + assert_eq!(result, Err(spacegate_kernel::observability::TelemetryError::MissingNamespace)); +} diff --git a/crates/shell/Cargo.toml b/crates/shell/Cargo.toml index 37098da6..d76a67c4 100644 --- a/crates/shell/Cargo.toml +++ b/crates/shell/Cargo.toml @@ -64,6 +64,12 @@ spacegate-ext-axum = { workspace = true, optional = true } regex = { workspace = true } futures-util.workspace = true tracing.workspace = true +tracing-subscriber = { workspace = true, features = ["env-filter"] } +tracing-opentelemetry.workspace = true +opentelemetry.workspace = true +opentelemetry_sdk.workspace = true +opentelemetry-otlp.workspace = true +opentelemetry-appender-tracing.workspace = true tokio.workspace = true hyper.workspace = true rustls-pemfile.workspace = true @@ -72,7 +78,6 @@ tokio-util = { workspace = true, features = ["io"] } [dev-dependencies] reqwest = { workspace = true } -tracing-subscriber = { workspace = true } criterion = { version = "0.5", features = ["async_tokio"] } testcontainers-modules = { workspace = true, features = ["redis"] } [package.metadata.docs.rs] diff --git a/crates/shell/src/config.rs b/crates/shell/src/config.rs index f72b8a8e..b92d39ce 100644 --- a/crates/shell/src/config.rs +++ b/crates/shell/src/config.rs @@ -29,6 +29,7 @@ where C: Retrieve + CreateListener + 'static, { let (init_config, listener) = config.create_listener().await?; + crate::observability::init(&init_config.observability); #[cfg(feature = "ext-axum")] let listener = { use crate::ext_features::axum::{shell_routers, App}; diff --git a/crates/shell/src/lib.rs b/crates/shell/src/lib.rs index 8bd47644..36c25626 100644 --- a/crates/shell/src/lib.rs +++ b/crates/shell/src/lib.rs @@ -49,6 +49,8 @@ use tracing::{info, instrument}; pub mod config; /// http extensions pub mod extension; +/// OpenTelemetry initialization. +pub mod observability; /// Spacegate service creation pub mod server; diff --git a/crates/shell/src/observability.rs b/crates/shell/src/observability.rs new file mode 100644 index 00000000..8ef285fd --- /dev/null +++ b/crates/shell/src/observability.rs @@ -0,0 +1,224 @@ +use std::sync::OnceLock; +use std::time::Duration; + +use opentelemetry::global; +use opentelemetry::trace::TracerProvider as _; +use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::{ + logs::SdkLoggerProvider, + metrics::{PeriodicReader, SdkMeterProvider}, + trace::{Sampler, SdkTracerProvider}, + Resource, +}; +use spacegate_config::model::{ObservabilityConfig, OtlpProtocol}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer}; + +static OTEL_GUARD: OnceLock = OnceLock::new(); + +#[derive(Debug)] +pub struct ObservabilityGuard { + tracer_provider: Option, + meter_provider: Option, + logger_provider: Option, +} + +impl ObservabilityGuard { + pub fn shutdown(&self) { + if let Some(provider) = &self.tracer_provider { + if let Err(err) = provider.shutdown() { + eprintln!("failed to shutdown otel tracer provider: {err}"); + } + } + if let Some(provider) = &self.meter_provider { + if let Err(err) = provider.shutdown() { + eprintln!("failed to shutdown otel meter provider: {err}"); + } + } + if let Some(provider) = &self.logger_provider { + if let Err(err) = provider.shutdown() { + eprintln!("failed to shutdown otel logger provider: {err}"); + } + } + } +} + +impl Drop for ObservabilityGuard { + fn drop(&mut self) { + self.shutdown(); + } +} + +pub fn init(config: &ObservabilityConfig) { + let _ = OTEL_GUARD.get_or_init(|| match build_guard(config) { + Ok(guard) => guard, + Err(err) => { + eprintln!("failed to initialize OpenTelemetry, falling back to stdout tracing: {err}"); + init_stdout_only(); + ObservabilityGuard { + tracer_provider: None, + meter_provider: None, + logger_provider: None, + } + } + }); +} + +fn build_guard(config: &ObservabilityConfig) -> Result> { + let env_filter = EnvFilter::from_default_env(); + let fmt_layer = tracing_subscriber::fmt::layer(); + if !config.enabled { + tracing_subscriber::registry().with(env_filter).with(fmt_layer).try_init()?; + return Ok(ObservabilityGuard { + tracer_provider: None, + meter_provider: None, + logger_provider: None, + }); + } + + let resource = Resource::builder().with_service_name(config.service_name.clone()).build(); + let mut guard = ObservabilityGuard { + tracer_provider: None, + meter_provider: None, + logger_provider: None, + }; + + let trace_layer = if config.traces.enabled { + match build_span_exporter(config) { + Ok(exporter) => { + let provider = SdkTracerProvider::builder() + .with_resource(resource.clone()) + .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(config.traces.sample_ratio)))) + .with_batch_exporter(exporter) + .build(); + let tracer = provider.tracer("spacegate"); + global::set_tracer_provider(provider.clone()); + guard.tracer_provider = Some(provider); + Some(tracing_opentelemetry::layer().with_tracer(tracer).boxed()) + } + Err(err) => { + eprintln!("failed to initialize OpenTelemetry traces, disabling traces: {err}"); + None + } + } + } else { + None + }; + + if config.metrics.enabled { + match build_metric_exporter(config) { + Ok(exporter) => { + let reader = PeriodicReader::builder(exporter).with_interval(metric_export_interval(config)).build(); + let provider = SdkMeterProvider::builder().with_resource(resource.clone()).with_reader(reader).build(); + global::set_meter_provider(provider.clone()); + guard.meter_provider = Some(provider); + } + Err(err) => { + eprintln!("failed to initialize OpenTelemetry metrics, disabling metrics: {err}"); + } + } + } + + let log_layer = if config.logs.enabled { + match build_log_exporter(config) { + Ok(exporter) => { + let provider = SdkLoggerProvider::builder().with_resource(resource).with_batch_exporter(exporter).build(); + let level_filter = log_level_filter(config); + let layer = OpenTelemetryTracingBridge::new(&provider).with_filter(level_filter).boxed(); + guard.logger_provider = Some(provider); + Some(layer) + } + Err(err) => { + eprintln!("failed to initialize OpenTelemetry logs, disabling logs: {err}"); + None + } + } + } else { + None + }; + + tracing_subscriber::registry().with(env_filter).with(fmt_layer).with(trace_layer).with(log_layer).try_init()?; + Ok(guard) +} + +fn init_stdout_only() { + let _ = tracing_subscriber::fmt().with_env_filter(tracing_subscriber::EnvFilter::from_default_env()).try_init(); +} + +fn otlp_protocol(config: &ObservabilityConfig) -> opentelemetry_otlp::Protocol { + match config.protocol { + OtlpProtocol::Grpc => opentelemetry_otlp::Protocol::Grpc, + OtlpProtocol::Http => opentelemetry_otlp::Protocol::HttpBinary, + } +} + +fn metric_export_interval(config: &ObservabilityConfig) -> Duration { + Duration::from_millis(config.metrics.export_interval_ms) +} + +fn log_level_filter(config: &ObservabilityConfig) -> tracing_subscriber::filter::LevelFilter { + config.logs.level.parse::().unwrap_or(tracing_subscriber::filter::LevelFilter::INFO) +} + +fn build_span_exporter(config: &ObservabilityConfig) -> Result { + let timeout = Duration::from_secs(5); + match config.protocol { + OtlpProtocol::Grpc => opentelemetry_otlp::SpanExporter::builder().with_tonic().with_endpoint(config.otlp_endpoint.clone()).with_timeout(timeout).build(), + OtlpProtocol::Http => { + opentelemetry_otlp::SpanExporter::builder().with_http().with_endpoint(config.otlp_endpoint.clone()).with_protocol(otlp_protocol(config)).with_timeout(timeout).build() + } + } +} + +#[cfg(test)] +mod tests { + use spacegate_config::model::{LogConfig, MetricConfig}; + + use super::*; + + #[test] + fn metric_export_interval_uses_configured_millis() { + let config = ObservabilityConfig { + metrics: MetricConfig { + enabled: true, + export_interval_ms: 15_000, + }, + ..Default::default() + }; + + assert_eq!(metric_export_interval(&config), Duration::from_secs(15)); + } + + #[test] + fn invalid_log_level_falls_back_to_info() { + let config = ObservabilityConfig { + logs: LogConfig { + enabled: true, + level: "not-a-level".to_string(), + }, + ..Default::default() + }; + + assert_eq!(log_level_filter(&config), tracing_subscriber::filter::LevelFilter::INFO); + } +} + +fn build_metric_exporter(config: &ObservabilityConfig) -> Result { + let timeout = Duration::from_secs(5); + match config.protocol { + OtlpProtocol::Grpc => opentelemetry_otlp::MetricExporter::builder().with_tonic().with_endpoint(config.otlp_endpoint.clone()).with_timeout(timeout).build(), + OtlpProtocol::Http => { + opentelemetry_otlp::MetricExporter::builder().with_http().with_endpoint(config.otlp_endpoint.clone()).with_protocol(otlp_protocol(config)).with_timeout(timeout).build() + } + } +} + +fn build_log_exporter(config: &ObservabilityConfig) -> Result { + let timeout = Duration::from_secs(5); + match config.protocol { + OtlpProtocol::Grpc => opentelemetry_otlp::LogExporter::builder().with_tonic().with_endpoint(config.otlp_endpoint.clone()).with_timeout(timeout).build(), + OtlpProtocol::Http => { + opentelemetry_otlp::LogExporter::builder().with_http().with_endpoint(config.otlp_endpoint.clone()).with_protocol(otlp_protocol(config)).with_timeout(timeout).build() + } + } +} diff --git a/crates/shell/src/server.rs b/crates/shell/src/server.rs index 899e21ea..8628a4de 100644 --- a/crates/shell/src/server.rs +++ b/crates/shell/src/server.rs @@ -293,14 +293,14 @@ impl RunningSgGateway { tls_server_cfg.alpn_protocols = vec![b"http/1.1".to_vec(), b"h2".to_vec()]; tls_server_cfg.ignore_client_order = true; tls_server_cfg.enable_secret_extraction = true; - listen.add_service(service.clone().https(tls_server_cfg)) + listen.add_service(service.clone().https_with_gateway_name(tls_server_cfg, gateway_name.clone())) } else { error!("[SG.Server] Can not found a valid Tls private key"); } }; } } else { - listen.add_service(service.clone().http()); + listen.add_service(service.clone().http_with_gateway_name(gateway_name.clone())); } listens.push(listen) } diff --git a/docs/otlp/otel-three-signals-guide.md b/docs/otlp/otel-three-signals-guide.md new file mode 100644 index 00000000..7b369eb2 --- /dev/null +++ b/docs/otlp/otel-three-signals-guide.md @@ -0,0 +1,445 @@ +# SpaceGate OTEL 三信号说明 + +本文说明 SpaceGate 当前接入 OpenTelemetry 后，`logs`、`traces`、`metrics` 三类数据分别如何上报、数据结构大致是什么样、以及分别适合哪些审计和监控需求。 + +## 1. 当前链路 + +当前本地验证链路： + +```text +SpaceGate + -> OTLP gRPC + -> OpenTelemetry Collector + -> ClickHouse +``` + +本地配置位置： + +```text +/tmp/spacegate-otel/config/config.json +/tmp/spacegate-otel/otel-collector.yaml +``` + +SpaceGate OTLP endpoint： + +```json +"otlp_endpoint": "http://127.0.0.1:4317", +"protocol": "grpc" +``` + +Collector 接收： + +```yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +``` + +Collector 写入 ClickHouse： + +```yaml +exporters: + clickhouse: + endpoint: tcp://spacegate-clickhouse:9000?dial_timeout=10s + database: otel + create_schema: true +``` + +## 2. Logs：审计明细 + +### 上报方式 + +SpaceGate 使用 Rust `tracing::info!` 生成结构化日志事件，再通过 OpenTelemetry logs exporter 走 OTLP 推送给 Collector。 + +当前每个请求完成时，网关会生成一条 `info` 级别的 access log： + +```text +event = "http_access" +``` + +插件可以把业务审计字段写入请求级 `TelemetryContext`，请求结束时统一进入 access log 的 `telemetry` 字段。 + +### 数据结构 + +ClickHouse 表： + +```text +otel_logs +``` + +常见字段形态： + +```text +Timestamp +TraceId +SpanId +SeverityText +Body +LogAttributes +ResourceAttributes +``` + +其中 `LogAttributes` 是 key/value map。当前 SpaceGate access log 关键字段： + +```text +LogAttributes['event'] = 'http_access' +LogAttributes['gateway'] = 'local' +LogAttributes['method'] = 'GET' +LogAttributes['path'] = '/' +LogAttributes['host'] = '127.0.0.1:9000' +LogAttributes['authority'] = '127.0.0.1:9000' +LogAttributes['client_ip'] = '127.0.0.1' +LogAttributes['x_forwarded_for'] +LogAttributes['user_agent'] = 'curl/8.7.1' +LogAttributes['downstream_remote_address'] = '127.0.0.1:xxxxx' +LogAttributes['route_name'] = 'local-test' +LogAttributes['upstream_host'] = '127.0.0.1' +LogAttributes['trace_id'] = '4bf92f3577b34da6a3ce929d0e0e4736' +LogAttributes['status_code'] = '200' +LogAttributes['request_id'] = '...' +LogAttributes['peer_addr'] = '127.0.0.1:xxxxx' +LogAttributes['duration_ms'] = '...' +LogAttributes['bytes_received'] +LogAttributes['bytes_sent'] +LogAttributes['request_body_size'] +LogAttributes['response_body_size'] +LogAttributes['telemetry'] = '{"ai.asset_id":"...","ai.total_tokens":"37"}' +``` + +`client_ip` 优先取 `X-Forwarded-For` 的第一个 IP，缺失时退回 TCP peer IP。MAC 地址不是 HTTP 请求语义的一部分，网关在代理、NAT、K8s 场景下无法可靠获得；如果审计确实需要，只能由上游可信组件或插件以业务字段写入 `telemetry`。 + +插件写入的审计字段在 `telemetry` JSON 里。例如： + +```json +{ + "ai.asset_id": "deepseek-chat", + "ai.asset_type": "model", + "ai.prompt_tokens": "24", + "ai.completion_tokens": "13", + "ai.total_tokens": "37", + "mcp.server": "search-service", + "mcp.tool": "web_search", + "auth.app_id": "demo-app" +} +``` + +查询示例： + +```sql +SELECT + Timestamp, + LogAttributes['request_id'] AS request_id, + LogAttributes['path'] AS path, + LogAttributes['status_code'] AS status_code, + JSONExtractString(LogAttributes['telemetry'], 'ai.asset_id') AS asset_id, + toUInt64OrZero(JSONExtractString(LogAttributes['telemetry'], 'ai.total_tokens')) AS total_tokens, + JSONExtractString(LogAttributes['telemetry'], 'mcp.tool') AS mcp_tool +FROM otel_logs +WHERE LogAttributes['event'] = 'http_access' +ORDER BY Timestamp DESC +LIMIT 20; +``` + +### 适合的需求 + +Logs 适合做**审计明细**： + +- 每次接口调用记录 +- 请求状态码、耗时、request_id +- 应用、API Key 摘要、租户信息 +- 大模型 asset_id、token 用量 +- MCP server/tool 调用信息 +- 错误码、失败原因 +- 审计中心按请求维度查询和导出 + +### 不适合的需求 + +Logs 不适合作为高频实时监控聚合的唯一数据源。虽然可以统计，但大量 JSON 提取和明细扫描成本较高。高频监控建议用 metrics。 + +## 3. Traces：调用链路 + +### 上报方式 + +SpaceGate 在请求入口创建 HTTP server span。插件内部使用 `tracing` 打出的事件可以挂到当前 span 上。OpenTelemetry traces exporter 通过 OTLP 推送给 Collector。 + +### 数据结构 + +ClickHouse 表： + +```text +otel_traces +``` + +常见字段形态： + +```text +Timestamp +TraceId +SpanId +ParentSpanId +SpanName +ServiceName +Duration +StatusCode +SpanAttributes +ResourceAttributes +Events +``` + +当前请求 span 示例字段： + +```text +SpanName = 'http.server.request' +SpanAttributes['http.method'] = 'GET' +SpanAttributes['http.path'] = '/' +SpanAttributes['http.host'] = '127.0.0.1:9000' +SpanAttributes['http.protocol'] = 'HTTP/1.1' +SpanAttributes['http.status_code'] = '200' +SpanAttributes['request_id'] = '...' +SpanAttributes['peer_addr'] = '127.0.0.1:xxxxx' +SpanAttributes['duration_ms'] = '...' +``` + +查询示例： + +```sql +SELECT + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + Duration, + StatusCode, + SpanAttributes['http.status_code'] AS http_status_code, + SpanAttributes['request_id'] AS request_id +FROM otel_traces +ORDER BY Timestamp DESC +LIMIT 20; +``` + +### 适合的需求 + +Traces 适合做**链路诊断**： + +- 一次请求经过了哪些内部阶段 +- 哪个插件或后端调用耗时高 +- 请求失败时定位失败发生在哪一段 +- 根据 `TraceId` 把 logs 和 spans 串起来 +- 抽样分析慢请求和异常请求 + +### 不适合的需求 + +Traces 不适合作为完整审计账本。生产环境通常会采样，例如 1%、0.1% 或 parent-based sampling。审计要求完整性时，应以 logs 为准。 + +## 4. Metrics：聚合监控 + +### 上报方式 + +SpaceGate 使用 OpenTelemetry metrics SDK 定期导出指标。当前本地配置里有： + +```json +"metrics": { + "enabled": true, + "export_interval_ms": 5000 +} +``` + +这表示每 5 秒导出一次当前指标数据。即使没有新请求，累计型指标也可能周期性写入 ClickHouse，所以 metrics 表行数会持续增加。 + +### 数据结构 + +ClickHouse 表通常包括： + +```text +otel_metrics_sum +otel_metrics_histogram +otel_metrics_gauge +otel_metrics_summary +otel_metrics_exp_histogram +``` + +当前 SpaceGate 请求级指标包括： + +```text +http.server.requests +http.server.errors +http.server.errors.4xx +http.server.errors.5xx +http.server.active_requests +http.server.request.duration +http.server.request.body.size +http.server.response.body.size +``` + +指标属性使用低基数字段： + +```text +gateway +http.request.method +http.response.status_code +network.protocol.name +network.protocol.version +``` + +示例含义： + +```text +http.server.requests + 类型：Counter + 作用：请求总量 + +http.server.request.duration + 类型：Histogram + 单位：s + 作用：请求耗时分布，可计算 P50/P95/P99 + +http.server.errors.5xx + 类型：Counter + 作用：服务端错误数量 + +http.server.active_requests + 类型：UpDownCounter + 作用：当前活跃请求数 +``` + +### 适合的需求 + +Metrics 适合做**监控和告警**： + +- QPS +- 错误率 +- P95/P99 延迟 +- 活跃请求数 +- 请求/响应大小分布 +- 4xx/5xx 趋势 +- 容量规划 +- SLO/SLA 面板 + +### 不适合的需求 + +Metrics 不适合做逐请求审计： + +- 不包含完整 request_id +- 不应该带 api_key、user_id、asset_id 这类高基数字段 +- 不记录每次请求的完整业务明细 +- 周期性导出会产生重复时间序列点，不能用行数代表请求数 + +## 5. 三者对比 + +| 信号 | 粒度 | 数据完整性 | 成本 | 主要用途 | ClickHouse 表 | +| --- | --- | --- | --- | --- | --- | +| Logs | 单请求明细 | 高 | 中到高 | 审计、账单、问题回溯 | `otel_logs` | +| Traces | 调用链路 | 取决于采样 | 中 | 慢请求诊断、链路分析 | `otel_traces` | +| Metrics | 聚合数据 | 聚合后数据 | 低到中 | 监控、告警、趋势 | `otel_metrics_*` | + +## 6. 审计中心推荐使用方式 + +审计中心建议以 logs 为主： + +```text +otel_logs + WHERE LogAttributes['event'] = 'http_access' +``` + +核心查询字段： + +```text +Timestamp +LogAttributes['request_id'] +LogAttributes['gateway'] +LogAttributes['method'] +LogAttributes['path'] +LogAttributes['status_code'] +LogAttributes['duration_ms'] +LogAttributes['telemetry'] +``` + +插件业务字段从 `telemetry` JSON 里解析： + +```sql +JSONExtractString(LogAttributes['telemetry'], 'ai.asset_id') +JSONExtractString(LogAttributes['telemetry'], 'ai.total_tokens') +JSONExtractString(LogAttributes['telemetry'], 'mcp.tool') +``` + +如果审计中心需要高频统计，例如按模型统计 token： + +```sql +SELECT + JSONExtractString(LogAttributes['telemetry'], 'ai.asset_id') AS asset_id, + sum(toUInt64OrZero(JSONExtractString(LogAttributes['telemetry'], 'ai.total_tokens'))) AS total_tokens +FROM otel_logs +WHERE LogAttributes['event'] = 'http_access' +GROUP BY asset_id; +``` + +生产上建议对常用字段建 ClickHouse 物化视图，把 JSON 字段抽成列，提升查询性能。 + +## 7. 监控系统推荐使用方式 + +监控面板建议使用 metrics： + +- `http.server.requests` 计算请求量 +- `http.server.errors` / `http.server.requests` 计算错误率 +- `http.server.request.duration` 计算延迟分位数 +- `http.server.active_requests` 观察并发压力 + +本地测试阶段如果只验证审计入库，可以关闭 metrics： + +```json +"metrics": { + "enabled": false, + "export_interval_ms": 60000 +} +``` + +生产建议使用较低频率： + +```json +"metrics": { + "enabled": true, + "export_interval_ms": 30000 +} +``` + +如果 metrics 长期写 ClickHouse，建议配置 TTL 或单独存入更适合时序数据的系统。 + +## 8. 推荐配置策略 + +### 本地审计验证 + +```text +logs: enabled +traces: enabled, sample_ratio = 1.0 +metrics: disabled +``` + +### 生产审计 + +```text +logs: enabled +traces: enabled, parent-based sampling +metrics: enabled, 30s 或 60s interval +``` + +### 生产监控 + +```text +metrics: enabled +logs: only access/audit logs +traces: sampling +``` + +## 9. 总结 + +- **Logs 是审计主数据**：每个请求一条 `http_access`，插件审计字段在 `telemetry` JSON 中。 +- **Traces 是诊断数据**：用 `TraceId` 追踪一次请求的链路和耗时。 +- **Metrics 是监控数据**：周期性聚合导出，用于 QPS、错误率、延迟、告警。 +- 不要把业务审计字段作为 metrics label。 +- 不要把 traces 当完整审计账本。 +- 审计中心主要查 `otel_logs`，监控系统主要用 `otel_metrics_*`。 diff --git a/docs/otlp/telemetry-pluginized-audit-plan.md b/docs/otlp/telemetry-pluginized-audit-plan.md new file mode 100644 index 00000000..49129614 --- /dev/null +++ b/docs/otlp/telemetry-pluginized-audit-plan.md @@ -0,0 +1,615 @@ +# Telemetry Pluginized Audit Fields Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 将请求级业务审计字段改造成真正插件化的 `TelemetryContext`：插件按命名空间写入字段，网关只负责统一携带、校验、序列化到 access log，并通过 OTLP logs 入库。 + +**Architecture:** `kernel` 提供请求级 `TelemetryContext` 和字段校验/序列化能力；`plugin` 提供原生插件写入 API；`service.rs` 只输出通用 access log 字段和一个 JSON 字符串 `telemetry`，不再硬编码 AI/MCP/token 等业务字段。ClickHouse 查询侧从 `LogAttributes['telemetry']` JSON 解析插件自定义字段。 + +**Tech Stack:** Rust, hyper request extensions, tracing structured logs, OpenTelemetry logs, ClickHouse `Map`/JSON extraction. + +--- + +## 设计补充与风险点 + +当前方向基本正确，但还需要明确这些边界： + +- **命名空间冲突**：A/B 插件不能直接共用 `total_tokens` 这类裸 key，推荐 `ai.total_tokens`、`mcp.tool`、`auth.app_id`。 +- **保留前缀**：禁止插件写 `http.*`、`net.*`、`gateway.*`、`spacegate.*`、`otel.*`，避免和网关/OTEL 主字段混淆。 +- **字段结构**：`TelemetryContext` 保持扁平 `key/value`，不接受嵌套 JSON 对象，避免合并语义和查询复杂度失控。 +- **字段长度**：限制 key/value 大小，防止插件误写完整 prompt、response body 或超大错误堆栈。 +- **覆盖策略**：同 key 后写覆盖前写；这是同命名空间内插件自己的责任。跨插件通过 namespace 避免冲突。 +- **敏感信息**：不建议写完整 `api_key`，推荐写 `api_key_hash` 或脱敏值。 +- **metrics 边界**：业务审计字段只进入 logs/traces，不作为 metrics label，避免高基数。 +- **ClickHouse 性能**：低频查询可直接 `JSONExtract*`；高频统计建议建物化视图抽取常用字段。 +- **WASM 对齐**：WASM host function 也必须遵守同一套 key 校验、namespace 和长度限制。 + +## File Structure + +- Modify: `crates/kernel/src/observability.rs` + - 定义 telemetry 字段校验规则。 + - 提供 `TelemetryError`。 + - 提供 `TelemetryContext::insert_checked`。 + - 提供 `TelemetryContext::insert_namespaced`。 + - 提供 `telemetry_json`. + +- Modify: `crates/kernel/src/service.rs` + - 删除硬编码 `telemetry.asset_id`、`telemetry.total_tokens` 等业务字段。 + - access log 只输出一个 `telemetry` JSON 字符串。 + +- Modify: `crates/plugin/src/lib.rs` + - 保留 `set_telemetry_field`，内部走 checked insert。 + - 新增推荐 API `set_plugin_telemetry_field(req, namespace, key, value)`。 + - 返回 `Result<(), TelemetryError>`，让插件可感知字段被拒绝。 + +- Modify: `crates/plugin/tests/test_telemetry.rs` + - 覆盖原生插件 API、命名空间 API、非法 key、保留前缀。 + +- Modify: `scripts/otel-local/query-access-logs.sh` + - 从 `LogAttributes['telemetry']` JSON 中解析字段，不再查询 `LogAttributes['telemetry.asset_id']`。 + +- Modify: `docs/wasm-telemetry-host-function-plan.md` + - 对齐本计划中的校验规则和 telemetry JSON 入库形态。 + +--- + +### Task 1: Kernel Telemetry Validation + +**Files:** +- Modify: `crates/kernel/src/observability.rs` + +- [ ] **Step 1: Write failing tests for validation** + +Add these tests inside `#[cfg(test)] mod tests` in `crates/kernel/src/observability.rs`: + +```rust +#[test] +fn telemetry_key_validation_accepts_namespaced_keys() { + assert!(super::validate_telemetry_key("ai.total_tokens").is_ok()); + assert!(super::validate_telemetry_key("mcp.tool-name").is_ok()); + assert!(super::validate_telemetry_key("auth.api_key_hash").is_ok()); +} + +#[test] +fn telemetry_key_validation_rejects_bad_keys() { + assert_eq!(super::validate_telemetry_key(""), Err(super::TelemetryError::EmptyKey)); + assert_eq!(super::validate_telemetry_key("total_tokens"), Err(super::TelemetryError::MissingNamespace)); + assert_eq!(super::validate_telemetry_key("ai total_tokens"), Err(super::TelemetryError::InvalidKey)); + assert_eq!(super::validate_telemetry_key("http.status_code"), Err(super::TelemetryError::ReservedPrefix)); + assert_eq!(super::validate_telemetry_key("spacegate.internal"), Err(super::TelemetryError::ReservedPrefix)); +} + +#[test] +fn telemetry_value_validation_rejects_oversized_values() { + let value = "x".repeat(super::MAX_TELEMETRY_VALUE_LEN + 1); + assert_eq!(super::validate_telemetry_value(&value), Err(super::TelemetryError::ValueTooLong)); +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +cargo test -p spacegate-kernel telemetry_key_validation 2>&1 | head -c 12000 +``` + +Expected: FAIL because `validate_telemetry_key`, `validate_telemetry_value`, `TelemetryError`, or `MAX_TELEMETRY_VALUE_LEN` are missing. + +- [ ] **Step 3: Implement validation** + +Add near `TelemetryContext` in `crates/kernel/src/observability.rs`: + +```rust +pub const MAX_TELEMETRY_KEY_LEN: usize = 128; +pub const MAX_TELEMETRY_VALUE_LEN: usize = 4096; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TelemetryError { + EmptyKey, + MissingNamespace, + ReservedPrefix, + InvalidKey, + KeyTooLong, + ValueTooLong, +} + +pub fn validate_telemetry_key(key: &str) -> Result<(), TelemetryError> { + if key.is_empty() { + return Err(TelemetryError::EmptyKey); + } + if key.len() > MAX_TELEMETRY_KEY_LEN { + return Err(TelemetryError::KeyTooLong); + } + if !key.contains('.') { + return Err(TelemetryError::MissingNamespace); + } + if ["http.", "net.", "gateway.", "spacegate.", "otel."].iter().any(|prefix| key.starts_with(prefix)) { + return Err(TelemetryError::ReservedPrefix); + } + if !key.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) { + return Err(TelemetryError::InvalidKey); + } + Ok(()) +} + +pub fn validate_telemetry_value(value: &str) -> Result<(), TelemetryError> { + if value.len() > MAX_TELEMETRY_VALUE_LEN { + return Err(TelemetryError::ValueTooLong); + } + Ok(()) +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: + +```bash +cargo test -p spacegate-kernel telemetry_key_validation 2>&1 | head -c 12000 +cargo test -p spacegate-kernel telemetry_value_validation 2>&1 | head -c 12000 +``` + +Expected: PASS. + +--- + +### Task 2: Checked TelemetryContext API + +**Files:** +- Modify: `crates/kernel/src/observability.rs` + +- [ ] **Step 1: Write failing tests for checked insertion** + +Add tests: + +```rust +#[test] +fn telemetry_context_checked_insert_rejects_invalid_key_without_mutating_context() { + let context = super::TelemetryContext::default(); + + let result = context.insert_checked("total_tokens", "37"); + + assert_eq!(result, Err(super::TelemetryError::MissingNamespace)); + assert!(context.snapshot().is_empty()); +} + +#[test] +fn telemetry_context_namespaced_insert_builds_stable_key() { + let context = super::TelemetryContext::default(); + + context.insert_namespaced("ai", "total_tokens", 37).expect("insert"); + + let fields = context.snapshot(); + assert_eq!(fields.get("ai.total_tokens").map(String::as_str), Some("37")); +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +cargo test -p spacegate-kernel telemetry_context_checked_insert 2>&1 | head -c 12000 +cargo test -p spacegate-kernel telemetry_context_namespaced_insert 2>&1 | head -c 12000 +``` + +Expected: FAIL because methods are missing. + +- [ ] **Step 3: Implement checked APIs** + +Replace or extend `impl TelemetryContext` with: + +```rust +impl TelemetryContext { + pub fn insert(&self, key: impl Into, value: impl Into) { + let Ok(mut fields) = self.fields.lock() else { + return; + }; + fields.insert(key.into(), value.into()); + } + + pub fn insert_checked(&self, key: impl Into, value: impl ToString) -> Result<(), TelemetryError> { + let key = key.into(); + let value = value.to_string(); + validate_telemetry_key(&key)?; + validate_telemetry_value(&value)?; + let Ok(mut fields) = self.fields.lock() else { + return Ok(()); + }; + fields.insert(key, value); + Ok(()) + } + + pub fn insert_namespaced(&self, namespace: &str, key: &str, value: impl ToString) -> Result<(), TelemetryError> { + self.insert_checked(format!("{namespace}.{key}"), value) + } + + pub fn snapshot(&self) -> BTreeMap { + self.fields.lock().map(|fields| fields.clone()).unwrap_or_default() + } + + pub fn is_empty(&self) -> bool { + self.fields.lock().map(|fields| fields.is_empty()).unwrap_or(true) + } +} +``` + +- [ ] **Step 4: Run tests** + +Run: + +```bash +cargo test -p spacegate-kernel telemetry_context_ 2>&1 | head -c 12000 +``` + +Expected: PASS. + +--- + +### Task 3: Access Log Uses Generic Telemetry JSON + +**Files:** +- Modify: `crates/kernel/src/observability.rs` +- Modify: `crates/kernel/src/service.rs` + +- [ ] **Step 1: Write failing JSON serialization test** + +Add test: + +```rust +#[test] +fn telemetry_json_serializes_plugin_defined_fields() { + let fields = BTreeMap::from([ + ("ai.asset_id".to_string(), "deepseek-chat".to_string()), + ("ai.total_tokens".to_string(), "37".to_string()), + ("mcp.tool".to_string(), "search".to_string()), + ]); + + let json = super::telemetry_json(&fields); + + assert!(json.contains("\"ai.asset_id\":\"deepseek-chat\"")); + assert!(json.contains("\"ai.total_tokens\":\"37\"")); + assert!(json.contains("\"mcp.tool\":\"search\"")); +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: + +```bash +cargo test -p spacegate-kernel telemetry_json_serializes_plugin_defined_fields 2>&1 | head -c 12000 +``` + +Expected: FAIL because `telemetry_json` is missing. + +- [ ] **Step 3: Add serde_json dependency if needed** + +Check `crates/kernel/Cargo.toml`. If `serde_json` is not already present, add: + +```toml +serde_json = { workspace = true } +``` + +- [ ] **Step 4: Implement telemetry_json** + +Add to `crates/kernel/src/observability.rs`: + +```rust +pub fn telemetry_json(fields: &BTreeMap) -> String { + serde_json::to_string(fields).unwrap_or_else(|_| "{}".to_string()) +} +``` + +- [ ] **Step 5: Refactor service.rs access log** + +In `crates/kernel/src/service.rs`: + +1. Replace import of `telemetry_log_attributes` with `telemetry_json`. +2. Delete all hardcoded `telemetry.asset_id`, `telemetry.total_tokens`, `telemetry.mcp_tool`, etc. +3. Emit only: + +```rust +let telemetry = telemetry_json(&access_log.telemetry); +tracing::info!( + event = "http_access", + gateway = %access_log.gateway, + method = %access_log.method, + path = %access_log.path, + host = %access_log.host, + protocol_name = %access_log.protocol_name, + protocol_version = %access_log.protocol_version, + status_code = access_log.status_code, + request_id = %access_log.request_id, + peer_addr = %access_log.peer_addr, + duration_ms = access_log.duration_ms, + request_body_size = ?access_log.request_body_size, + response_body_size = ?access_log.response_body_size, + telemetry = %telemetry, + "http access log" +); +``` + +- [ ] **Step 6: Run tests** + +Run: + +```bash +cargo test -p spacegate-kernel observability::tests 2>&1 | head -c 12000 +``` + +Expected: PASS. + +--- + +### Task 4: Plugin API Becomes Namespaced and Checked + +**Files:** +- Modify: `crates/plugin/src/lib.rs` +- Modify: `crates/plugin/tests/test_telemetry.rs` + +- [ ] **Step 1: Update plugin tests** + +Replace `crates/plugin/tests/test_telemetry.rs` content with: + +```rust +use spacegate_plugin::{set_plugin_telemetry_field, set_telemetry_field, SgBody}; + +fn request_with_telemetry() -> hyper::Request { + let mut req = hyper::Request::builder().body(SgBody::empty()).expect("request"); + req.extensions_mut().insert(spacegate_kernel::observability::TelemetryContext::default()); + req +} + +#[test] +fn set_telemetry_field_writes_checked_request_context() { + let req = request_with_telemetry(); + + set_telemetry_field(&req, "ai.asset_id", "deepseek-chat").expect("insert"); + set_telemetry_field(&req, "ai.total_tokens", 37).expect("insert"); + + let fields = req.extensions().get::().expect("telemetry context").snapshot(); + assert_eq!(fields.get("ai.asset_id").map(String::as_str), Some("deepseek-chat")); + assert_eq!(fields.get("ai.total_tokens").map(String::as_str), Some("37")); +} + +#[test] +fn set_plugin_telemetry_field_adds_namespace() { + let req = request_with_telemetry(); + + set_plugin_telemetry_field(&req, "mcp", "tool", "search").expect("insert"); + + let fields = req.extensions().get::().expect("telemetry context").snapshot(); + assert_eq!(fields.get("mcp.tool").map(String::as_str), Some("search")); +} + +#[test] +fn set_telemetry_field_rejects_unqualified_key() { + let req = request_with_telemetry(); + + let result = set_telemetry_field(&req, "total_tokens", 37); + + assert_eq!(result, Err(spacegate_kernel::observability::TelemetryError::MissingNamespace)); +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +cargo test -p spacegate-plugin test_telemetry 2>&1 | head -c 12000 +``` + +Expected: FAIL because `set_telemetry_field` currently returns `()` and `set_plugin_telemetry_field` is missing. + +- [ ] **Step 3: Implement plugin APIs** + +In `crates/plugin/src/lib.rs`, replace current `set_telemetry_field` with: + +```rust +pub fn set_telemetry_field( + req: &SgRequest, + key: impl Into, + value: impl ToString, +) -> Result<(), spacegate_kernel::observability::TelemetryError> { + if let Some(context) = req.extensions().get::() { + context.insert_checked(key, value)?; + } + Ok(()) +} + +pub fn set_plugin_telemetry_field( + req: &SgRequest, + namespace: &str, + key: &str, + value: impl ToString, +) -> Result<(), spacegate_kernel::observability::TelemetryError> { + if let Some(context) = req.extensions().get::() { + context.insert_namespaced(namespace, key, value)?; + } + Ok(()) +} +``` + +- [ ] **Step 4: Run tests** + +Run: + +```bash +cargo test -p spacegate-plugin test_telemetry 2>&1 | head -c 12000 +``` + +Expected: PASS. + +--- + +### Task 5: Update ClickHouse Query Script + +**Files:** +- Modify: `scripts/otel-local/query-access-logs.sh` + +- [ ] **Step 1: Update SQL to parse telemetry JSON** + +Replace selected telemetry fields with: + +```sql +JSONExtractString(LogAttributes['telemetry'], 'ai.asset_id') AS ai_asset_id, +JSONExtractString(LogAttributes['telemetry'], 'ai.asset_type') AS ai_asset_type, +JSONExtractString(LogAttributes['telemetry'], 'ai.total_tokens') AS ai_total_tokens, +JSONExtractString(LogAttributes['telemetry'], 'mcp.server') AS mcp_server, +JSONExtractString(LogAttributes['telemetry'], 'mcp.tool') AS mcp_tool, +JSONExtractString(LogAttributes['telemetry'], 'mcp.success') AS mcp_success, +JSONExtractString(LogAttributes['telemetry'], 'auth.app_id') AS auth_app_id, +JSONExtractString(LogAttributes['telemetry'], 'auth.api_key_hash') AS auth_api_key_hash +``` + +Keep base fields: + +```sql +Timestamp, +Body, +SeverityText, +LogAttributes['event'] AS event, +LogAttributes['gateway'] AS gateway, +LogAttributes['method'] AS method, +LogAttributes['path'] AS path, +LogAttributes['status_code'] AS status_code, +LogAttributes['request_id'] AS request_id, +LogAttributes['duration_ms'] AS duration_ms, +LogAttributes['telemetry'] AS telemetry +``` + +- [ ] **Step 2: Validate shell syntax** + +Run: + +```bash +bash -n scripts/otel-local/query-access-logs.sh +``` + +Expected: no output and exit code 0. + +--- + +### Task 6: Update WASM Plan + +**Files:** +- Modify: `docs/wasm-telemetry-host-function-plan.md` + +- [ ] **Step 1: Align ABI plan with namespaced telemetry** + +Update the plan so `spacegate_set_telemetry_field` requires a fully qualified key: + +```text +ai.total_tokens +mcp.tool +auth.app_id +``` + +And add optional convenience SDK wrapper: + +```rust +pub fn set_plugin_telemetry_field(namespace: &str, key: &str, value: impl ToString) -> Result<(), Status> { + set_telemetry_field(&format!("{namespace}.{key}"), value) +} +``` + +- [ ] **Step 2: Align validation section** + +Document the same rules: + +- key max 128 bytes +- value max 4096 bytes +- key must contain `.` +- allowed key chars: `[A-Za-z0-9_.-]` +- reserved prefixes rejected: `http.`, `net.`, `gateway.`, `spacegate.`, `otel.` + +--- + +### Task 7: Full Verification + +**Files:** +- No code changes. + +- [ ] **Step 1: Format touched Rust files** + +Run: + +```bash +rustfmt --edition 2021 crates/kernel/src/observability.rs crates/kernel/src/service.rs crates/plugin/src/lib.rs crates/plugin/tests/test_telemetry.rs +``` + +Expected: no output and exit code 0. + +- [ ] **Step 2: Run targeted tests** + +Run: + +```bash +cargo test -p spacegate-kernel observability::tests 2>&1 | head -c 12000 +cargo test -p spacegate-plugin test_telemetry 2>&1 | head -c 12000 +``` + +Expected: PASS. + +- [ ] **Step 3: Run integration compile check** + +Run: + +```bash +cargo check -p spacegate-shell --features fs,plugin-wasm 2>&1 | head -c 12000 +``` + +Expected: PASS. Existing unrelated warnings may remain. + +- [ ] **Step 4: Validate local scripts** + +Run: + +```bash +for f in scripts/otel-local/*.sh; do bash -n "$f" || exit 1; done +``` + +Expected: no output and exit code 0. + +--- + +## Expected Result + +插件写： + +```rust +set_plugin_telemetry_field(&req, "ai", "asset_id", "deepseek-chat")?; +set_plugin_telemetry_field(&req, "ai", "total_tokens", 37)?; +set_plugin_telemetry_field(&req, "mcp", "tool", "search")?; +``` + +access log 入库： + +```text +LogAttributes['event'] = 'http_access' +LogAttributes['telemetry'] = '{"ai.asset_id":"deepseek-chat","ai.total_tokens":"37","mcp.tool":"search"}' +``` + +审计查询： + +```sql +SELECT + Timestamp, + LogAttributes['request_id'] AS request_id, + JSONExtractString(LogAttributes['telemetry'], 'ai.asset_id') AS asset_id, + toUInt64OrZero(JSONExtractString(LogAttributes['telemetry'], 'ai.total_tokens')) AS total_tokens, + JSONExtractString(LogAttributes['telemetry'], 'mcp.tool') AS mcp_tool +FROM otel_logs +WHERE LogAttributes['event'] = 'http_access' +ORDER BY Timestamp DESC; +``` + +## Self-Review + +- Spec coverage: covers namespace, validation, generic JSON access log, plugin API, ClickHouse query, WASM plan alignment. +- Placeholder scan: no TBD/TODO placeholders. +- Type consistency: `TelemetryError` lives in kernel and is returned by plugin APIs; `TelemetryContext` remains request extension. +- Boundary check: no AI/MCP/token business semantics remain in `service.rs`; those appear only in docs/scripts as examples. diff --git a/docs/otlp/wasm-telemetry-host-function-plan.md b/docs/otlp/wasm-telemetry-host-function-plan.md new file mode 100644 index 00000000..d831b902 --- /dev/null +++ b/docs/otlp/wasm-telemetry-host-function-plan.md @@ -0,0 +1,251 @@ +# WASM 插件审计字段 Host Function 技术方案 + +## 目标 + +让 WASM 插件也能像原生插件一样写入请求级业务审计字段，例如： + +- `ai.asset_id` +- `ai.asset_type` +- `ai.prompt_tokens` +- `ai.completion_tokens` +- `ai.total_tokens` +- `auth.app_id` +- `auth.api_key_hash` +- `mcp.server` +- `mcp.tool` +- `mcp.success` +- `error.code` + +这些字段最终随请求结束时的 `http_access` 日志进入 OTLP logs，再由 Collector 写入 ClickHouse 的 `otel_logs`。 + +## 当前原生插件链路 + +原生插件调用： + +```rust +spacegate_plugin::set_plugin_telemetry_field(&req, "ai", "asset_id", "deepseek-chat")?; +spacegate_plugin::set_plugin_telemetry_field(&req, "ai", "total_tokens", 37)?; +``` + +数据流： + +```text +SgRequest.extensions.TelemetryContext + -> kernel 请求结束生成 http_access 日志 + -> telemetry JSON log attribute + -> OTLP logs + -> Collector + -> ClickHouse otel_logs +``` + +## 推荐 WASM ABI + +新增非 proxy-wasm 标准的 SpaceGate 扩展 host function： + +```text +env.spacegate_set_telemetry_field(key_ptr, key_len, value_ptr, value_len) -> status +``` + +参数： + +- `key_ptr: i32` +- `key_len: i32` +- `value_ptr: i32` +- `value_len: i32` + +返回： + +- `Status::Ok` +- `Status::BadArgument` +- `Status::InvalidMemoryAccess` +- `Status::NotFound` + +命名选择： + +- 不复用 `proxy_call_foreign_function`，避免把核心审计能力塞进不透明 FFI。 +- 使用 `spacegate_` 前缀，明确这是 SpaceGate 扩展，不污染 proxy-wasm 标准 ABI。 + +## Host 侧实现 + +### 1. HostState 增加请求级 telemetry 存储 + +在 `crates/plugin-wasm/src/host_state.rs` 的 `RequestContext` 增加： + +```rust +pub telemetry_fields: BTreeMap, +``` + +原因： + +- WASM `Vm::process` 目前会把 `SgRequest` 拆成 `parts/body`，再重建 `new_req` 给 `inner.call`。 +- host fn 执行期间拿不到原始 `SgRequest` 引用。 +- 因此 WASM 调 host fn 时先写到当前 `RequestContext`，请求结束或调用 inner 前再同步到 `SgRequest.extensions.TelemetryContext`。 + +### 2. 注册 host function + +在 `crates/plugin-wasm/src/host_fn.rs` 增加： + +```rust +fn register_spacegate_telemetry(linker: &mut Linker) -> Result<(), wasmtime::Error> +``` + +并在 `register_all` 中调用。 + +处理逻辑： + +1. 用 `MemoryHelper::from_caller` 读取 guest memory。 +2. 读取 `key` 和 `value` 字符串。 +3. 校验： + - key 非空 + - key 最大 128 字节 + - value 最大 4096 字节 + - key 必须包含命名空间分隔符 `.` + - key 只能包含 `[A-Za-z0-9_.-]` + - 禁止保留前缀：`http.`、`net.`、`gateway.`、`spacegate.`、`otel.` +4. 获取 `caller.data_mut().current_context_mut()`。 +5. 写入 `ctx.telemetry_fields.insert(key, value)`。 +6. 返回 `Status::Ok`。 + +### 3. 同步到 SgRequest + +在 `crates/plugin-wasm/src/vm.rs` 的 `Vm::process` 中： + +- 重建 `new_req` 后、调用 `inner.call(new_req).await` 前： + +```rust +if let Some(kernel_ctx) = new_req.extensions().get::() { + for (key, value) in wasm_ctx.telemetry_fields { + kernel_ctx.insert_checked(key, value)?; + } +} +``` + +注意： + +- 需要把 `let new_req = ...` 改成 `let mut new_req = ...` 或在构造前保留 extensions。 +- 当前 request parts 来自原始 `SgRequest`，extensions 会保留，所以 kernel 插入的 `TelemetryContext` 可以继续存在。 + +### 4. 本地响应短路场景 + +如果 WASM 在 request 阶段通过 `proxy_send_local_response` 直接返回，不会调用 `inner.call`。 + +这种情况下也需要把 telemetry 同步回 access log： + +- 方案 A：短路前直接从原始 request extensions 同步。 +- 方案 B：在 `Vm::process` 开始时把 `TelemetryContext` clone 存进 `HostState` 或当前 `RequestContext`。 + +推荐方案 B： + +```rust +RequestContext { + telemetry_sink: Option, +} +``` + +在 `Vm::process` 开始时： + +```rust +let telemetry_sink = parts.extensions.get::().cloned(); +ctx.telemetry_sink = telemetry_sink; +``` + +host fn 写字段时： + +```rust +if let Some(sink) = &ctx.telemetry_sink { + sink.insert_checked(key.clone(), value.clone())?; +} +ctx.telemetry_fields.insert(key, value); +``` + +这样即使本地响应短路，kernel 请求结束时也能读到审计字段。 + +## Guest SDK 封装 + +WASM 插件侧建议提供一个薄封装： + +```rust +#[link(wasm_import_module = "env")] +extern "C" { + fn spacegate_set_telemetry_field( + key_ptr: i32, + key_len: i32, + value_ptr: i32, + value_len: i32, + ) -> i32; +} + +pub fn set_telemetry_field(key: &str, value: impl ToString) -> Result<(), Status> { + let value = value.to_string(); + let status = unsafe { + spacegate_set_telemetry_field( + key.as_ptr() as i32, + key.len() as i32, + value.as_ptr() as i32, + value.len() as i32, + ) + }; + Status::from_i32(status) +} + +pub fn set_plugin_telemetry_field(namespace: &str, key: &str, value: impl ToString) -> Result<(), Status> { + set_telemetry_field(&format!("{namespace}.{key}"), value) +} +``` + +插件使用： + +```rust +set_plugin_telemetry_field("ai", "asset_id", "deepseek-chat")?; +set_plugin_telemetry_field("ai", "prompt_tokens", 24)?; +set_plugin_telemetry_field("ai", "completion_tokens", 13)?; +set_plugin_telemetry_field("ai", "total_tokens", 37)?; +set_plugin_telemetry_field("mcp", "tool", "search")?; +``` + +## 测试计划 + +### 单元测试 + +- `host_fn` 能读取 guest memory 中的 key/value。 +- 非法 key 返回 `BadArgument`。 +- 空 key 返回 `BadArgument`。 +- 超长 value 返回 `BadArgument`。 +- 无当前 HTTP context 返回 `NotFound`。 + +### WASM 集成测试 + +新增一个测试 wasm： + +- 在 `proxy_on_request_headers` 写 `ai.asset_id`。 +- 在 `proxy_on_response_body` 写 token 字段。 +- 请求结束后断言 `TelemetryContext.snapshot()` 包含这些字段。 + +### 端到端测试 + +本地脚本启动： + +```bash +scripts/otel-local/start-clickhouse.sh +scripts/otel-local/start-collector.sh +scripts/otel-local/start-mock-ac.sh +scripts/otel-local/start-spacegate.sh +scripts/otel-local/request.sh +scripts/otel-local/query-access-logs.sh +``` + +确认 ClickHouse `otel_logs` 中包含： + +```text +JSONExtractString(LogAttributes['telemetry'], 'ai.asset_id') +JSONExtractString(LogAttributes['telemetry'], 'ai.total_tokens') +JSONExtractString(LogAttributes['telemetry'], 'mcp.tool') +``` + +## 风险与边界 + +- 这是 SpaceGate 扩展 ABI，不是 proxy-wasm 标准函数。 +- 字段 key 必须限制字符集和长度，避免 ClickHouse 查询侧难以治理。 +- 不建议把 `request_id`、用户 ID、完整 prompt、完整 response body 写入 telemetry 字段。 +- token、MCP、模型 ID 属于审计日志字段，不应作为 metrics label。 +- WASM 当前单 VM 串行处理请求，字段必须存放在 `RequestContext`，不能放全局 map。 From d7a9273bd309e9c45ddbb1bc123fdc5014c8cf05 Mon Sep 17 00:00:00 2001 From: jianxin5335 <51434929+jianxin5335@users.noreply.github.com> Date: Mon, 25 May 2026 14:32:58 +0800 Subject: [PATCH 19/19] feat: K8s infra-only stack, Harbor OCI plugin and build docs Add admin UI manifests and infra-only kustomize overlay, wire SgFilter to Harbor OCI wasm URL, extend build-images/verify scripts, and document OCI artifact publishing in ai-gateway-queue README. Co-authored-by: Cursor --- deploy/k8s/ai-gateway/admin-ui.yaml | 165 ++++++++++++++++++ deploy/k8s/ai-gateway/apply-infra.sh | 57 ++++++ deploy/k8s/ai-gateway/build-images.sh | 54 ++++-- .../k8s/ai-gateway/kustomization-infra.yaml | 22 +++ .../ai-gateway/sgfilter-ai-gateway-queue.yaml | 8 +- .../ai-gateway/spacegate-rbac-cluster.yaml | 28 +++ deploy/k8s/ai-gateway/verify-infra.sh | 73 ++++++++ deploy/k8s/ai-gateway/verify.sh | 15 +- plugins/wasm/ai-gateway-queue/README.md | 106 +++++++++++ 9 files changed, 506 insertions(+), 22 deletions(-) create mode 100644 deploy/k8s/ai-gateway/admin-ui.yaml create mode 100755 deploy/k8s/ai-gateway/apply-infra.sh create mode 100644 deploy/k8s/ai-gateway/kustomization-infra.yaml create mode 100644 deploy/k8s/ai-gateway/spacegate-rbac-cluster.yaml create mode 100755 deploy/k8s/ai-gateway/verify-infra.sh diff --git a/deploy/k8s/ai-gateway/admin-ui.yaml b/deploy/k8s/ai-gateway/admin-ui.yaml new file mode 100644 index 00000000..07e3cc5a --- /dev/null +++ b/deploy/k8s/ai-gateway/admin-ui.yaml @@ -0,0 +1,165 @@ +# SpaceGate Admin UI（K8s 模式） +# admin-server 读写 K8s 中的 Gateway / HTTPRoute / SgFilter 等 +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spacegate-admin + namespace: spacegate + labels: + app.kubernetes.io/part-of: ai-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: spacegate-admin + labels: + app.kubernetes.io/part-of: ai-gateway +rules: + - apiGroups: [""] + resources: [services, pods, secrets, configmaps] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: [apps] + resources: [daemonsets] + verbs: [get, list, watch] + - apiGroups: [gateway.networking.k8s.io] + resources: [gatewayclasses, gateways, httproutes, httproutes/status, gateways/status, gatewayclasses/status] + verbs: [get, list, create, update, watch, delete] + - apiGroups: [spacegate.idealworld.group] + resources: [sgfilters, httpspaceroutes, httpspaceroutes/status] + verbs: [get, create, update, patch, list, watch, delete] + - apiGroups: [extensions.higress.io] + resources: [wasmplugins, wasmplugins/status] + verbs: [get, create, update, patch, list, watch, delete] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: spacegate-admin + labels: + app.kubernetes.io/part-of: ai-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: spacegate-admin +subjects: + - kind: ServiceAccount + name: spacegate-admin + namespace: spacegate +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spacegate-admin + namespace: spacegate + labels: + app.kubernetes.io/name: spacegate-admin + app.kubernetes.io/part-of: ai-gateway +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: spacegate-admin + template: + metadata: + labels: + app.kubernetes.io/name: spacegate-admin + app.kubernetes.io/part-of: ai-gateway + spec: + serviceAccountName: spacegate-admin + containers: + - name: admin-server + image: ai-gateway/admin-server:dev + imagePullPolicy: IfNotPresent + args: + - -c + - k8s:spacegate + - -p + - "19992" + - -H + - 0.0.0.0 + env: + - name: CONFIG + value: k8s:spacegate + - name: RUST_LOG + value: info + ports: + - containerPort: 19992 + name: http + readinessProbe: + tcpSocket: + port: http + initialDelaySeconds: 3 + periodSeconds: 5 + resources: + requests: + cpu: 50m + memory: 64Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: spacegate-admin + namespace: spacegate + labels: + app.kubernetes.io/name: spacegate-admin +spec: + selector: + app.kubernetes.io/name: spacegate-admin + ports: + - name: http + port: 19992 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ai-gateway-web + namespace: spacegate + labels: + app.kubernetes.io/name: ai-gateway-web + app.kubernetes.io/part-of: ai-gateway +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: ai-gateway-web + template: + metadata: + labels: + app.kubernetes.io/name: ai-gateway-web + app.kubernetes.io/part-of: ai-gateway + spec: + containers: + - name: web + # 与 build-images.sh 输出一致；本地构建后 kubectl set image 更新 + image: ai-gateway/web:k8s-spa + imagePullPolicy: Never + ports: + - containerPort: 9080 + name: http + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 2 + periodSeconds: 5 + resources: + requests: + cpu: 10m + memory: 32Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: ai-gateway-web + namespace: spacegate + labels: + app.kubernetes.io/name: ai-gateway-web +spec: + type: LoadBalancer + selector: + app.kubernetes.io/name: ai-gateway-web + ports: + - name: http + port: 9080 + targetPort: http diff --git a/deploy/k8s/ai-gateway/apply-infra.sh b/deploy/k8s/ai-gateway/apply-infra.sh new file mode 100755 index 00000000..a418a0ae --- /dev/null +++ b/deploy/k8s/ai-gateway/apply-infra.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# 编译 Wasm + 部署 AI Gateway K8s 基础设施（不含默认 HTTPRoute ai-api / SgFilter） +set -euo pipefail +DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$DIR/../../.." && pwd)" +KUSTOMIZE_FILE="$DIR/kustomization-infra.yaml" +WASM_SRC="$ROOT/plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm" +WASM_DST="$DIR/files/spacegate_plugin_ai_gateway_queue.wasm" + +echo "==> 检查 SpaceGate 前置（namespace / GatewayClass / DaemonSet）" +if ! kubectl get namespace spacegate >/dev/null 2>&1; then + echo "ERROR: namespace 'spacegate' 不存在。请先执行：" >&2 + echo " ./scripts/deploy.sh k8s install-prereq" >&2 + echo " 或 kubectl apply -f $ROOT/resource/kube-manifests/" >&2 + exit 1 +fi + +echo "==> 移除默认 Demo 路由（若存在）" +kubectl delete -f "$DIR/httproute-ai.yaml" -n spacegate --ignore-not-found +kubectl delete -f "$DIR/sgfilter-ai-gateway-queue.yaml" -n spacegate --ignore-not-found + +echo "==> 编译 ai-gateway-queue Wasm" +cd "$ROOT" +rustup target add wasm32-wasip1 2>/dev/null || true +cargo build --release --target wasm32-wasip1 \ + --manifest-path plugins/wasm/Cargo.toml \ + -p spacegate_plugin_ai_gateway_queue + +mkdir -p "$DIR/files" +cp "$WASM_SRC" "$WASM_DST" + +echo "==> 应用 Kustomize（infra-only，无 ai-api HTTPRoute）" +KUST_BACKUP="$DIR/kustomization.yaml.full.bak" +cp "$DIR/kustomization.yaml" "$KUST_BACKUP" +cp "$DIR/kustomization-infra.yaml" "$DIR/kustomization.yaml" +kubectl apply -k "$DIR" +mv "$KUST_BACKUP" "$DIR/kustomization.yaml" + +echo "==> 确保 SpaceGate DaemonSet 使用 K8s 模式本地镜像" +SG_IMAGE="${SPACEGATE_K8S_IMAGE:-ai-gateway/spacegate:k8s}" +if kubectl get daemonset spacegate -n spacegate >/dev/null 2>&1; then + kubectl set image daemonset/spacegate spacegate="$SG_IMAGE" -n spacegate + kubectl rollout status daemonset/spacegate -n spacegate --timeout=180s +fi + +echo "==> 等待 AI Gateway Pod Ready" +kubectl wait --for=condition=ready pod \ + -l 'app.kubernetes.io/name in (ai-gateway-redis,ai-gateway-service,ai-gateway-wasm,ai-gateway-mock-upstream)' \ + -n spacegate \ + --timeout=180s + +echo "" +echo "部署完成（无默认 ai-api 路由）。" +echo " 验证: $DIR/verify-infra.sh" +echo "" +echo "后续：在管理界面或 kubectl 自行创建 HTTPRoute，并挂载 SgFilter / Wasm 插件。" +echo " Gateway 入口: ai-gateway（:9993，SpaceGate hostNetwork）" diff --git a/deploy/k8s/ai-gateway/build-images.sh b/deploy/k8s/ai-gateway/build-images.sh index 466e75f8..e933f87d 100755 --- a/deploy/k8s/ai-gateway/build-images.sh +++ b/deploy/k8s/ai-gateway/build-images.sh @@ -1,18 +1,46 @@ #!/usr/bin/env bash -# 构建 ai-gateway-service Linux 镜像（供 K8s 使用） +# 构建 K8s 所需镜像：ai-gateway-service + SpaceGate（k8s 模式） set -euo pipefail -ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" -cd "$ROOT" +DIR="$(cd "$(dirname "$0")" && pwd)" +SG_ROOT="$(cd "$DIR/../../.." && pwd)" +# ai-gateway-dev 工作区根（spacegate 的父目录） +WORKSPACE_ROOT="$(cd "$SG_ROOT/.." && pwd)" -IMAGE="${AI_GATEWAY_SERVICE_IMAGE:-ai-gateway/service:dev}" -DOCKERFILE="$(dirname "$0")/docker/Dockerfile.ai-gateway-service" +SERVICE_IMAGE="${AI_GATEWAY_SERVICE_IMAGE:-ai-gateway/service:dev}" +SG_IMAGE="${SPACEGATE_K8S_IMAGE:-ai-gateway/spacegate:k8s}" -echo "Building ai-gateway-service (linux/amd64) ..." -docker build -f "$DOCKERFILE" \ - --build-arg SPACEGATE_ROOT="$ROOT" \ - -t "$IMAGE" \ - "$ROOT" +echo "==> 构建 ai-gateway-service" +docker build -f "$DIR/docker/Dockerfile.ai-gateway-service" \ + --build-arg SPACEGATE_ROOT="$SG_ROOT" \ + -t "$SERVICE_IMAGE" \ + "$SG_ROOT" -echo "Done. Image: $IMAGE" -echo "For k3d/minikube: import locally, e.g." -echo " k3d image import $IMAGE -c " +echo "==> 构建 SpaceGate（wasm + axum + k8s）" +docker build -f "$WORKSPACE_ROOT/docker/Dockerfile.spacegate-k8s" \ + -t "$SG_IMAGE" \ + "$SG_ROOT" + +WEB_IMAGE="${AI_GATEWAY_WEB_IMAGE:-ai-gateway/web:k8s-spa}" +echo "==> 构建管理 UI（spacegate-admin-fe SPA + nginx）" +if [[ ! -f "$WORKSPACE_ROOT/spacegate-admin-fe/dist/index.html" ]]; then + echo " 缺少 dist/index.html，尝试构建前端（需已 npm install）" + (cd "$WORKSPACE_ROOT/spacegate-admin-fe" && VITE_AI_GATEWAY_BASE_URL=/ai-gateway npm run build) || { + echo " 前端构建失败，请手动: cd spacegate-admin-fe && VITE_AI_GATEWAY_BASE_URL=/ai-gateway npm run build" + exit 1 + } +fi +docker build -f "$WORKSPACE_ROOT/docker/Dockerfile.web.k8s" \ + -t "$WEB_IMAGE" \ + "$WORKSPACE_ROOT" + +echo "Done." +echo " ai-gateway-service: $SERVICE_IMAGE" +echo " spacegate (k8s): $SG_IMAGE" +echo " admin web (k8s): $WEB_IMAGE" +echo "" +echo "更新管理 UI Deployment（本地 Docker Desktop 需 Never 拉取策略）：" +echo " kubectl set image deployment/ai-gateway-web web=$WEB_IMAGE -n spacegate" +echo " kubectl rollout status deployment/ai-gateway-web -n spacegate" +echo "" +echo "更新 DaemonSet 镜像（若已安装 SpaceGate）：" +echo " kubectl set image daemonset/spacegate spacegate=$SG_IMAGE -n spacegate" diff --git a/deploy/k8s/ai-gateway/kustomization-infra.yaml b/deploy/k8s/ai-gateway/kustomization-infra.yaml new file mode 100644 index 00000000..fdbbcabc --- /dev/null +++ b/deploy/k8s/ai-gateway/kustomization-infra.yaml @@ -0,0 +1,22 @@ +# 基础设施栈：不含默认 HTTPRoute ai-api / SgFilter +# 用法：kubectl apply -k . --kustomize-file kustomization-infra.yaml（见 apply-infra.sh） +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: spacegate + +resources: + - redis.yaml + - mock-upstream.yaml + - wasm-server.yaml + - ai-gateway-service.yaml + - gateway-ai.yaml + - spacegate-rbac-cluster.yaml + - admin-ui.yaml + +configMapGenerator: + - name: ai-gateway-queue-wasm + files: + - files/spacegate_plugin_ai_gateway_queue.wasm + options: + disableNameSuffixHash: true diff --git a/deploy/k8s/ai-gateway/sgfilter-ai-gateway-queue.yaml b/deploy/k8s/ai-gateway/sgfilter-ai-gateway-queue.yaml index 1d496f54..0a934017 100644 --- a/deploy/k8s/ai-gateway/sgfilter-ai-gateway-queue.yaml +++ b/deploy/k8s/ai-gateway/sgfilter-ai-gateway-queue.yaml @@ -16,7 +16,13 @@ spec: name: ai-gateway-queue enable: true config: - url: http://ai-gateway-wasm/spacegate_plugin_ai_gateway_queue.wasm + # Harbor OCI 制品（本地 push 脚本见 open-source/harbor/push-ai-gateway-queue.sh） + url: oci+http://host.docker.internal:9081/ai-gateway/ai-gateway-queue:v1.0.0 + sha256: sha256:8e2b1d3271b7e7c44b01e96e1844e0a231896fbe21b23ba07a01f71a58b9e697 + oci_auth: + registry: host.docker.internal:9081 + username: admin + password: Harbor12345 fail_strategy: fail_close validate_on_create: false plugin_name: ai-gateway-queue diff --git a/deploy/k8s/ai-gateway/spacegate-rbac-cluster.yaml b/deploy/k8s/ai-gateway/spacegate-rbac-cluster.yaml new file mode 100644 index 00000000..5b753f06 --- /dev/null +++ b/deploy/k8s/ai-gateway/spacegate-rbac-cluster.yaml @@ -0,0 +1,28 @@ +# 补充 SpaceGate ServiceAccount 对 SgFilter / Gateway API 的集群级 list 权限 +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: spacegate-k8s-config +rules: + - apiGroups: ["gateway.networking.k8s.io"] + resources: ["gateways", "httproutes", "gateways/status", "httproutes/status"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["spacegate.idealworld.group"] + resources: ["sgfilters", "httpspaceroutes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["extensions.higress.io"] + resources: ["wasmplugins"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: spacegate-k8s-config +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: spacegate-k8s-config +subjects: + - kind: ServiceAccount + name: spacegate + namespace: spacegate diff --git a/deploy/k8s/ai-gateway/verify-infra.sh b/deploy/k8s/ai-gateway/verify-infra.sh new file mode 100755 index 00000000..443f2ccf --- /dev/null +++ b/deploy/k8s/ai-gateway/verify-infra.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# 基础设施部署验证（不要求 HTTPRoute / 网关流量） +set -euo pipefail +NS=spacegate +pass=0 +fail=0 + +check() { + local name="$1" expect="$2" got="$3" + if [[ "$got" == "$expect" ]]; then + echo "✅ $name" + pass=$((pass + 1)) + else + echo "❌ $name 期望=$expect 实际=$got" + fail=$((fail + 1)) + fi +} + +echo "==> Pod 状态" +kubectl get pods -n "$NS" -l 'app.kubernetes.io/name in (ai-gateway-redis,ai-gateway-service,ai-gateway-wasm,ai-gateway-mock-upstream)' + +echo "==> 不应存在默认 HTTPRoute ai-api" +if kubectl get httproute ai-api -n "$NS" >/dev/null 2>&1; then + echo "❌ HTTPRoute ai-api 仍存在（应已删除）" + fail=$((fail + 1)) +else + echo "✅ 无 HTTPRoute ai-api" + pass=$((pass + 1)) +fi + +echo "==> 不应存在默认 SgFilter ai-gateway-queue" +if kubectl get sgfilter ai-gateway-queue -n "$NS" >/dev/null 2>&1; then + echo "❌ SgFilter ai-gateway-queue 仍存在" + fail=$((fail + 1)) +else + echo "✅ 无 SgFilter ai-gateway-queue" + pass=$((pass + 1)) +fi + +echo "==> Gateway ai-gateway 存在" +kubectl get gateway ai-gateway -n "$NS" >/dev/null && check "Gateway ai-gateway" "ok" "ok" || fail=$((fail + 1)) + +echo "==> SpaceGate DaemonSet 运行中" +if kubectl get pods -n "$NS" -l app=spacegate -o jsonpath='{.items[0].status.phase}' 2>/dev/null | grep -q Running; then + echo "✅ spacegate DaemonSet Running" + pass=$((pass + 1)) +else + echo "❌ spacegate DaemonSet 未 Running" + fail=$((fail + 1)) +fi + +echo "==> ai-gateway-service 健康（集群内 curl）" +if kubectl run curl-health-$RANDOM --rm -i --restart=Never -n "$NS" \ + --image=curlimages/curl:8.5.0 --quiet -- \ + curl -sf http://ai-gateway-service:18080/healthz >/dev/null 2>&1; then + echo "✅ ai-gateway-service /healthz" + pass=$((pass + 1)) +else + echo "❌ ai-gateway-service /healthz" + fail=$((fail + 1)) +fi + +echo "==> Wasm HTTP 分发" +if kubectl exec -n "$NS" deploy/ai-gateway-wasm -- wget -qO- http://127.0.0.1/spacegate_plugin_ai_gateway_queue.wasm >/dev/null 2>&1; then + echo "✅ ai-gateway-wasm 可下载 .wasm" + pass=$((pass + 1)) +else + echo "❌ ai-gateway-wasm" + fail=$((fail + 1)) +fi + +echo "=== $pass 通过, $fail 失败 ===" +exit "$fail" diff --git a/deploy/k8s/ai-gateway/verify.sh b/deploy/k8s/ai-gateway/verify.sh index b63df86c..5256d512 100755 --- a/deploy/k8s/ai-gateway/verify.sh +++ b/deploy/k8s/ai-gateway/verify.sh @@ -4,6 +4,7 @@ set -euo pipefail DIR="$(cd "$(dirname "$0")" && pwd)" NS=spacegate GW="http://127.0.0.1:9993/v1/chat/completions" +PF="" pass=0 fail=0 @@ -25,22 +26,20 @@ curl -sf "http://127.0.0.1:18080/healthz" >/dev/null 2>&1 \ && echo "✅ ai-gateway-service /healthz（集群内）" \ || { echo "⚠️ 跳过直连 health（请 kubectl port-forward svc/ai-gateway-service 18080:18080）"; } -T="k8s-verify-$RANDOM" +T="k8s-verify-$(date +%s)" curl -sf -X PUT "http://127.0.0.1:18080/v1/admin/tenant-rate-limits" \ -H 'Content-Type: application/json' \ - -d "{\"tenant\":\"$T\",\"rps\":5,\"burst\":10}" >/dev/null 2>&1 \ - || kubectl port-forward -n "$NS" svc/ai-gateway-service 18080:18080 >/tmp/pf.log 2>&1 & -PF=$! -sleep 2 + -d "{\"tenant\":\"$T\",\"rps\":5,\"burst\":5}" >/dev/null 2>&1 \ + || { kubectl port-forward -n "$NS" svc/ai-gateway-service 18080:18080 >/tmp/pf-18080.log 2>&1 & PF=$!; sleep 2; } curl -sf -X PUT "http://127.0.0.1:18080/v1/admin/tenant-rate-limits" \ -H 'Content-Type: application/json' \ - -d "{\"tenant\":\"$T\",\"rps\":5,\"burst\":10}" >/dev/null || true + -d "{\"tenant\":\"$T\",\"rps\":5,\"burst\":5}" >/dev/null || true -echo "==> 网关插件（tenant=$T）" +echo "==> 网关插件 (tenant=$T)" check "缺 Policy" 400 "$(curl -s -o /dev/null -w '%{http_code}' -X POST "$GW" -H "X-Tenant-Id: $T" -H 'Content-Type: application/json' -d '{}')" check "abandon 配额内" 200 "$(curl -s -o /dev/null -w '%{http_code}' -X POST "$GW" -H 'X-RateLimit-Policy: abandon' -H "X-Tenant-Id: $T" -H 'Content-Type: application/json' -d '{"p":1}')" -for i in $(seq 1 8); do +for i in $(seq 1 10); do curl -s -o /dev/null -X POST "$GW" -H 'X-RateLimit-Policy: abandon' -H "X-Tenant-Id: $T" -H 'Content-Type: application/json' -d "{\"p\":$i}" || true done check "abandon 超额" 429 "$(curl -s -o /dev/null -w '%{http_code}' -X POST "$GW" -H 'X-RateLimit-Policy: abandon' -H "X-Tenant-Id: $T" -H 'Content-Type: application/json' -d '{"p":99}')" diff --git a/plugins/wasm/ai-gateway-queue/README.md b/plugins/wasm/ai-gateway-queue/README.md index 5045bda3..9ae621b8 100644 --- a/plugins/wasm/ai-gateway-queue/README.md +++ b/plugins/wasm/ai-gateway-queue/README.md @@ -47,6 +47,112 @@ cargo build --release --target wasm32-wasip1 --manifest-path plugins/wasm/Cargo. plugins/wasm/target/wasm32-wasip1/release/spacegate_plugin_ai_gateway_queue.wasm ``` +## 制作 OCI 制品 + +生产环境建议将 `.wasm` 以 **OCI Artifact** 形式推送到镜像仓库（Harbor、GHCR、ACR 等），SpaceGate 通过 `oci://` / `oci+http://` URL 拉取，而不是挂载本地文件或 HTTP 分发服务。 + +> OCI 制品是单层 Wasm 文件，用 **oras** 推送，不是 `docker build` 容器镜像。Docker Hub 通常不支持 Wasm OCI Artifact，请用 Harbor / GHCR / ACR 等。 + +### 前置条件 + +```bash +# 安装 oras（OCI 推送/拉取工具） +brew install oras + +# 确保已编译 wasm（见上一节「构建」） +rustup target add wasm32-wasip1 +``` + +### 方式一：推送到本地 Harbor（推荐联调） + +本地 Harbor 示例：`http://localhost:9081`，默认账号 `admin` / `Harbor12345`。 + +**手动推送**： + +```bash +WASM_DIR=plugins/wasm/target/wasm32-wasip1/release +WASM=spacegate_plugin_ai_gateway_queue.wasm + +# 1. 创建 Harbor 项目（已存在可跳过） +curl -u 'admin:Harbor12345' -X POST 'http://localhost:9081/api/v2.0/projects' \ + -H 'Content-Type: application/json' \ + -d '{"project_name":"ai-gateway","public":false}' + +# 2. 登录 Harbor（HTTP 需加 --plain-http） +echo 'Harbor12345' | oras login localhost:9081 -u admin --password-stdin --plain-http + +# 3. 计算 digest（生产建议写入插件配置） +shasum -a 256 "$WASM_DIR/$WASM" + +# 4. 推送 OCI Artifact +cd "$WASM_DIR" +oras push localhost:9081/ai-gateway/ai-gateway-queue:v1.0.0 --plain-http \ + --artifact-type application/vnd.module.wasm.content.layer.v1+wasm \ + "${WASM}:application/wasm" +``` + +推送成功后可在 Harbor UI **项目 → ai-gateway → ai-gateway-queue** 查看制品。 + +### 方式二：推送到任意 OCI 仓库（GHCR / ACR / 私有 Harbor） + +在 `spacegate` 仓库根目录： + +```bash +# 登录目标仓库 +oras login ghcr.io -u YOUR_USER + +# 使用仓库自带脚本 +REGISTRY=ghcr.io/your-org TAG=v1.0.0 ./deploy/push-wasm-oci.sh + +# 或从 ai-gateway-dev 工作区根目录 +REGISTRY=ghcr.io/your-org ./scripts/deploy.sh oci push +``` + +### 插件配置中引用 OCI + +SpaceGate 支持的 URL 形式： + +| 场景 | 示例 | +|------|------| +| HTTPS 仓库 | `oci://ghcr.io/your-org/ai-gateway-queue:v1.0.0` | +| 本地 Harbor（HTTP） | `oci+http://localhost:9081/ai-gateway/ai-gateway-queue:v1.0.0` | +| K8s 拉取宿主机 Harbor（Docker Desktop） | `oci+http://host.docker.internal:9081/ai-gateway/ai-gateway-queue:v1.0.0` | + +完整配置示例（含校验与私有仓库凭证）可参考工作区 `open-source/harbor/plugins/wasm.ai-gateway-queue.oci.json`（与 `ai-gateway-dev` 同级目录下的 Harbor 联调配置）： + +```json +{ + "url": "oci+http://host.docker.internal:9081/ai-gateway/ai-gateway-queue:v1.0.0", + "sha256": "sha256:<编译产物 shasum -a 256 输出>", + "oci_auth": { + "registry": "host.docker.internal:9081", + "username": "admin", + "password": "Harbor12345" + }, + "fail_strategy": "fail_close", + "plugin_name": "ai-gateway-queue", + "plugin_config": { "...": "..." }, + "clusters": { + "ai-gateway-service": "http://ai-gateway-service:18080" + } +} +``` + +K8s **SgFilter** 中在 `spec.filters[].config` 写入上述字段即可；`clusters` 仍指向 `ai-gateway-service`（与 Wasm 存储方式无关）。 + +### 版本更新 + +1. 修改代码后重新 `cargo build --release --target wasm32-wasip1 ...` +2. 用新 tag 执行 `oras push`（如 `v1.0.1`） +3. 更新 SpaceGate 配置中的 `url` 与 `sha256`（或 `module_cache_key`） + +### 注意事项 + +- **`sha256`**：建议生产开启，防止同 tag 被覆盖后加载错误版本 +- **私有仓库**：配置 `oci_auth`，或 K8s WasmPlugin 使用 `imagePullSecret` +- **K8s 网络**：Pod 内勿用 `localhost:9081` 指宿主机 Harbor，Docker Desktop 用 `host.docker.internal:9081` +- 更多细节见 [`deploy/README.md`](../../../deploy/README.md) 第 7 节 + ## 启动外部服务 `ai-gateway-queue` 依赖外部服务来完成限流、入队、等待和回调。