diff --git a/xllm/core/distributed_runtime/comm_channel.cpp b/xllm/core/distributed_runtime/comm_channel.cpp index a4240da4bc..a2622ae714 100644 --- a/xllm/core/distributed_runtime/comm_channel.cpp +++ b/xllm/core/distributed_runtime/comm_channel.cpp @@ -24,9 +24,10 @@ limitations under the License. namespace xllm { -bool CommChannel::init_brpc(const std::string& server_address) { +bool CommChannel::init_brpc(const std::string& server_address, + int32_t timeout_ms) { options_.connection_type = "pooled"; - options_.timeout_ms = -1; + options_.timeout_ms = timeout_ms; options_.connect_timeout_ms = -1; options_.max_retry = 3; @@ -53,14 +54,12 @@ bool CommChannel::hello() { return true; } -bool CommChannel::check_health() { +bool CommChannel::check_health(int32_t timeout_ms) { proto::Status req; proto::Status resp; brpc::Controller cntl; - // Set a timeout for health check - // check hang status: 10min(magic num) - cntl.set_timeout_ms(600000); + cntl.set_timeout_ms(timeout_ms); stub_->Hello(&cntl, &req, &resp, nullptr); if (cntl.Failed()) { LOG(WARNING) << "Health check failed: " << cntl.ErrorText(); diff --git a/xllm/core/distributed_runtime/comm_channel.h b/xllm/core/distributed_runtime/comm_channel.h index 06e632a09e..eb9909d46e 100644 --- a/xllm/core/distributed_runtime/comm_channel.h +++ b/xllm/core/distributed_runtime/comm_channel.h @@ -19,6 +19,7 @@ limitations under the License. #include #include +#include #include #include #include @@ -38,7 +39,7 @@ class CommChannel { CommChannel() = default; virtual ~CommChannel() = default; - bool init_brpc(const std::string& server_address); + bool init_brpc(const std::string& server_address, int32_t timeout_ms = -1); virtual bool hello(); @@ -115,7 +116,7 @@ class CommChannel { folly::Promise& promise); // Check if the connection to worker is healthy - virtual bool check_health(); + virtual bool check_health(int32_t timeout_ms = 600000); virtual bool sleep(MasterStatus master_status); diff --git a/xllm/core/distributed_runtime/dit_engine.cpp b/xllm/core/distributed_runtime/dit_engine.cpp index 6a64357d82..9801508263 100644 --- a/xllm/core/distributed_runtime/dit_engine.cpp +++ b/xllm/core/distributed_runtime/dit_engine.cpp @@ -18,13 +18,23 @@ limitations under the License. #include #include +#include + +#include +#include +#include +#include +#include #include "common/device_monitor.h" #include "core/common/global_flags.h" #include "core/common/metrics.h" #include "core/distributed_runtime/master.h" +#include "core/framework/config/dit_config.h" #include "core/framework/config/execution_config.h" #include "core/platform/device.h" +#include "distributed_runtime/comm_channel.h" +#include "distributed_runtime/remote_worker.h" #include "framework/parallel_state/parallel_args.h" #include "framework/parallel_state/parallel_state.h" #include "runtime/worker.h" @@ -32,6 +42,17 @@ limitations under the License. #include "util/timer.h" namespace xllm { + +namespace { + +int64_t monotonic_time_ms() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} + +} // namespace + DiTEngine::DiTEngine(const runtime::Options& options, std::shared_ptr dist_manager) : options_(options), dist_manager_(dist_manager) { @@ -59,6 +80,7 @@ DiTEngine::DiTEngine(const runtime::Options& options, // setup all workers and create worker clients in nnode_rank=0 engine side. setup_workers(options); worker_clients_num_ = worker_clients_.size(); + setup_vae_workers(); // init thread pool threadpool_ = std::make_unique( @@ -74,6 +96,185 @@ void DiTEngine::setup_workers(const runtime::Options& options) { worker_clients_ = dist_manager_->get_worker_clients(); } +void DiTEngine::setup_vae_workers() { + const auto& config = DiTConfig::get_instance(); + if (config.dit_instance_role() != "dit") { + return; + } + + const std::string& model_id = options_.model_id(); + const bool is_flux_model = model_id == "flux" || model_id == "flux-dev" || + model_id.find("flux-dev-") == 0; + CHECK(is_flux_model) + << "Separate DiT/VAE instances currently support Flux only, got model: " + << model_id; + + CHECK(!config.dit_vae_service_addresses().empty()) + << "dit_vae_service_addresses must be set for a dit instance."; + + std::stringstream addresses(config.dit_vae_service_addresses()); + std::unordered_set configured_addresses; + std::string address; + int32_t rank = 0; + while (std::getline(addresses, address, ',')) { + const size_t first = address.find_first_not_of(" \t\n\r"); + const size_t last = address.find_last_not_of(" \t\n\r"); + if (first == std::string::npos) { + continue; + } + address = address.substr(first, last - first + 1); + CHECK(configured_addresses.insert(address).second) + << "Duplicate VAE service address: " << address; + auto channel = std::make_unique(); + CHECK(channel->init_brpc(address, config.dit_vae_request_timeout_ms())) + << "Failed to connect to VAE service: " << address; + auto worker_state = std::make_unique(); + worker_state->worker = std::make_shared( + rank++, address, options_.devices().front(), std::move(channel)); + vae_workers_.emplace_back(std::move(worker_state)); + } + CHECK(!vae_workers_.empty()) + << "No valid VAE service address was configured."; + const size_t route_seed = static_cast(getpid()) ^ + static_cast(config.dit_worker_port()); + next_vae_worker_.store(route_seed, std::memory_order_relaxed); + LOG(INFO) << "Configured " << vae_workers_.size() + << " VAE service instance(s) for DiT routing."; +} + +size_t DiTEngine::select_vae_worker( + const std::vector& attempted_workers) { + CHECK_EQ(attempted_workers.size(), vae_workers_.size()); + + const size_t start_index = + next_vae_worker_.fetch_add(1, std::memory_order_relaxed) % + vae_workers_.size(); + for (size_t offset = 0; offset < vae_workers_.size(); ++offset) { + const size_t worker_index = (start_index + offset) % vae_workers_.size(); + if (attempted_workers[worker_index]) { + continue; + } + return worker_index; + } + + LOG(FATAL) << "No untried VAE worker is available."; + return 0; +} + +DiTForwardOutput DiTEngine::decode_with_vae( + const DiTForwardInput& input, + const DiTForwardOutput& latent_output) { + if (latent_output.tensors.empty()) { + LOG(ERROR) << "DiT instance returned no latent tensors."; + return {}; + } + + DiTForwardInput vae_input = input; + vae_input.prompts.clear(); + vae_input.prompts_2.clear(); + vae_input.negative_prompts.clear(); + vae_input.negative_prompts_2.clear(); + vae_input.prompt_embeds = torch::Tensor(); + vae_input.pooled_prompt_embeds = torch::Tensor(); + vae_input.negative_prompt_embeds = torch::Tensor(); + vae_input.negative_pooled_prompt_embeds = torch::Tensor(); + vae_input.images = torch::Tensor(); + vae_input.images_list.clear(); + vae_input.mask_images = torch::Tensor(); + vae_input.control_image = torch::Tensor(); + vae_input.masked_image_latents = torch::Tensor(); + vae_input.last_images = torch::Tensor(); + if (latent_output.tensors.size() == 1) { + vae_input.latents = latent_output.tensors.front(); + } else { + vae_input.latents = torch::cat(latent_output.tensors, 0); + } + + ForwardInput forward_input; + forward_input.input_params.dit_forward_input = std::move(vae_input); + std::vector attempted_workers(vae_workers_.size(), false); + const bool debug_print = DiTConfig::get_instance().dit_debug_print(); + Timer decode_timer; + for (size_t offset = 0; offset < vae_workers_.size(); ++offset) { + const size_t worker_index = select_vae_worker(attempted_workers); + attempted_workers[worker_index] = true; + auto& worker_state = *vae_workers_[worker_index]; + const auto& config = DiTConfig::get_instance(); + const int64_t now_ms = monotonic_time_ms(); + if (!worker_state.healthy.load(std::memory_order_relaxed) && + now_ms < + worker_state.next_health_check_ms.load(std::memory_order_relaxed)) { + continue; + } + if (!worker_state.healthy.load(std::memory_order_relaxed) && + !worker_state.worker->check_health( + config.dit_vae_health_check_timeout_ms())) { + worker_state.next_health_check_ms.store( + now_ms + config.dit_vae_health_check_interval_ms(), + std::memory_order_relaxed); + continue; + } + worker_state.healthy.store(true, std::memory_order_relaxed); + worker_state.next_health_check_ms.store(0, std::memory_order_relaxed); + Timer rpc_timer; + std::optional result; + try { + result = vae_workers_[worker_index] + ->worker->step_remote_async(forward_input) + .get(); + } catch (const std::exception& exception) { + worker_state.healthy.store(false, std::memory_order_relaxed); + worker_state.next_health_check_ms.store( + monotonic_time_ms() + config.dit_vae_health_check_interval_ms(), + std::memory_order_relaxed); + LOG(WARNING) << "VAE worker " << worker_index + << " threw while decoding latent output: " + << exception.what() << "; trying next worker."; + continue; + } catch (...) { + worker_state.healthy.store(false, std::memory_order_relaxed); + worker_state.next_health_check_ms.store( + monotonic_time_ms() + config.dit_vae_health_check_interval_ms(), + std::memory_order_relaxed); + LOG(WARNING) << "VAE worker " << worker_index + << " threw an unknown exception while decoding latent " + "output; trying next worker."; + continue; + } + if (!result.has_value()) { + worker_state.healthy.store(false, std::memory_order_relaxed); + worker_state.next_health_check_ms.store( + monotonic_time_ms() + config.dit_vae_health_check_interval_ms(), + std::memory_order_relaxed); + LOG(WARNING) << "VAE worker " << worker_index + << " failed to decode latent output, trying next worker."; + continue; + } + const auto& output = result->dit_forward_output; + if (output.tensors.size() != input.batch_size) { + worker_state.healthy.store(false, std::memory_order_relaxed); + worker_state.next_health_check_ms.store( + monotonic_time_ms() + config.dit_vae_health_check_interval_ms(), + std::memory_order_relaxed); + LOG(WARNING) << "VAE worker " << worker_index + << " returned an invalid tensor count: " + << output.tensors.size() << ", expected " << input.batch_size + << "."; + continue; + } + if (debug_print) { + LOG(INFO) << "VAE worker " << worker_index + << " decode rpc latency: " << rpc_timer.elapsed_seconds() + << " s, total latency: " << decode_timer.elapsed_seconds() + << " s."; + } + return output; + } + + LOG(ERROR) << "All VAE workers failed to decode latent output."; + return {}; +} + bool DiTEngine::init() { if (!init_model()) { LOG(ERROR) << "Failed to init model from: " << options_.model_path(); @@ -134,10 +335,28 @@ DiTForwardOutput DiTEngine::step(std::vector& batches) { auto results = folly::collectAll(futures).get(); // return the result from the driver + for (const auto& result : results) { + if (result.hasException() || !result.value().has_value()) { + LOG(ERROR) << "At least one DiT worker failed to execute the request."; + batches[0].process_forward_error( + Status(StatusCode::UNAVAILABLE, + "A DiT worker failed to execute the request.")); + return {}; + } + } auto forward_output = results.front().value(); - DCHECK(forward_output.has_value()) << "Failed to execute model"; - batches[0].process_forward_output(forward_output.value().dit_forward_output); - return forward_output.value().dit_forward_output; + DiTForwardOutput output = forward_output.value().dit_forward_output; + if (DiTConfig::get_instance().dit_instance_role() == "dit") { + output = decode_with_vae(dit_forward_input, output); + if (output.tensors.empty()) { + batches[0].process_forward_error( + Status(StatusCode::UNAVAILABLE, + "All configured VAE workers failed to decode the request.")); + return output; + } + } + batches[0].process_forward_output(output); + return output; } std::vector DiTEngine::get_active_activation_memory() const { diff --git a/xllm/core/distributed_runtime/dit_engine.h b/xllm/core/distributed_runtime/dit_engine.h index 663f67b782..35a87b97fe 100644 --- a/xllm/core/distributed_runtime/dit_engine.h +++ b/xllm/core/distributed_runtime/dit_engine.h @@ -18,10 +18,13 @@ limitations under the License. #include +#include #include +#include #include "common/macros.h" #include "dist_manager.h" +#include "distributed_runtime/remote_worker.h" #include "engine.h" #include "framework/batch/dit_batch.h" #include "framework/parallel_state/process_group.h" @@ -76,6 +79,10 @@ class DiTEngine : public Engine { private: // setup workers internal void setup_workers(const runtime::Options& options); + void setup_vae_workers(); + size_t select_vae_worker(const std::vector& attempted_workers); + DiTForwardOutput decode_with_vae(const DiTForwardInput& input, + const DiTForwardOutput& latent_output); // init models bool init_model(); // options @@ -84,6 +91,13 @@ class DiTEngine : public Engine { int64_t worker_clients_num_; // a list of process groups, with each process group handling a single device std::vector> process_groups_; + struct VaeWorkerState { + std::shared_ptr worker; + std::atomic healthy{true}; + std::atomic next_health_check_ms{0}; + }; + std::vector> vae_workers_; + std::atomic next_vae_worker_{0}; }; } // namespace xllm diff --git a/xllm/core/distributed_runtime/remote_worker.cpp b/xllm/core/distributed_runtime/remote_worker.cpp index 9f576b7e33..d15675b6ef 100644 --- a/xllm/core/distributed_runtime/remote_worker.cpp +++ b/xllm/core/distributed_runtime/remote_worker.cpp @@ -335,7 +335,9 @@ folly::SemiFuture RemoteWorker::get_active_activation_memory_async() { return future; } -bool RemoteWorker::check_health() { return channel_->check_health(); } +bool RemoteWorker::check_health(int32_t timeout_ms) { + return channel_->check_health(timeout_ms); +} folly::SemiFuture RemoteWorker::sleep_async(MasterStatus master_status) { folly::Promise promise; diff --git a/xllm/core/distributed_runtime/remote_worker.h b/xllm/core/distributed_runtime/remote_worker.h index 2a7132ec6c..efb5d685bc 100644 --- a/xllm/core/distributed_runtime/remote_worker.h +++ b/xllm/core/distributed_runtime/remote_worker.h @@ -139,7 +139,7 @@ class RemoteWorker : public WorkerClient { folly::SemiFuture get_active_activation_memory_async() override; // Check if the connection to worker is healthy - bool check_health(); + bool check_health(int32_t timeout_ms = 600000); // Get worker global rank int32_t global_rank() const { return global_rank_; } diff --git a/xllm/core/distributed_runtime/worker_server.cpp b/xllm/core/distributed_runtime/worker_server.cpp index 9b25a3851c..b8c0e22fd6 100644 --- a/xllm/core/distributed_runtime/worker_server.cpp +++ b/xllm/core/distributed_runtime/worker_server.cpp @@ -36,6 +36,7 @@ limitations under the License. #include "common/metrics.h" #include "core/distributed_runtime/spawn_worker_server/spawn_worker_protocol.h" +#include "core/framework/config/dit_config.h" #include "core/framework/config/eplb_config.h" #include "core/framework/config/execution_config.h" #include "core/framework/config/kernel_config.h" @@ -141,20 +142,34 @@ void WorkerServer::create_server(const runtime::Options& options, } auto worker_server = ServerRegistry::get_instance().register_server(server_name_); - if (!worker_server->start(worker_service, addr + ":0")) { + int32_t worker_port = 0; + if (options.backend() == "dit") { + worker_port = DiTConfig::get_instance().dit_worker_port(); + if (worker_port > 0) { + const int64_t rank_port = static_cast(worker_port) + + static_cast(worker_global_rank); + CHECK_LE(rank_port, 65535) + << "dit_worker_port plus global rank exceeds the valid port range: " + << rank_port; + worker_port = static_cast(rank_port); + } + } + const std::string worker_server_addr = + addr + ":" + std::to_string(worker_port); + if (!worker_server->start(worker_service, worker_server_addr)) { LOG(ERROR) << "failed to start distribute worker server on address: " << addr; return; } - auto worker_server_addr = + const std::string actual_worker_server_addr = addr + ":" + std::to_string(worker_server->listen_port()); LOG(INFO) << "Worker " << worker_global_rank - << ": server address: " << worker_server_addr; + << ": server address: " << actual_worker_server_addr; // Sync with master node proto::AddressInfo addr_info; - addr_info.set_address(worker_server_addr); + addr_info.set_address(actual_worker_server_addr); addr_info.set_global_rank(worker_global_rank); proto::CommUniqueIdList uids; if (!sync_master_node(master_node_addr, addr_info, uids)) { diff --git a/xllm/core/framework/batch/dit_batch.cpp b/xllm/core/framework/batch/dit_batch.cpp index b06702b895..91a9807bd8 100644 --- a/xllm/core/framework/batch/dit_batch.cpp +++ b/xllm/core/framework/batch/dit_batch.cpp @@ -256,4 +256,10 @@ void DiTBatch::process_forward_output(const DiTForwardOutput& output) { } } +void DiTBatch::process_forward_error(const Status& status) { + for (const auto& request : request_vec_) { + request->handle_error(status); + } +} + } // namespace xllm diff --git a/xllm/core/framework/batch/dit_batch.h b/xllm/core/framework/batch/dit_batch.h index be0f66f8cc..c069129fd3 100644 --- a/xllm/core/framework/batch/dit_batch.h +++ b/xllm/core/framework/batch/dit_batch.h @@ -40,6 +40,8 @@ struct DiTBatch { void process_forward_output(const DiTForwardOutput& output); + void process_forward_error(const Status& status); + private: std::vector> request_vec_; }; diff --git a/xllm/core/framework/config/dit_config.cpp b/xllm/core/framework/config/dit_config.cpp index c213c36e32..c0a1b698c4 100644 --- a/xllm/core/framework/config/dit_config.cpp +++ b/xllm/core/framework/config/dit_config.cpp @@ -15,6 +15,8 @@ limitations under the License. #include "core/framework/config/dit_config.h" +#include + #include "core/common/global_flags.h" #include "core/framework/config/config_utils.h" @@ -53,6 +55,30 @@ DEFINE_int64(dit_cache_end_blocks, 5, "The number of blocks to skip at the end."); +DEFINE_string(dit_instance_role, "all", "DiT instance role: all, dit, or vae."); + +DEFINE_string(dit_vae_service_addresses, + "", + "Comma-separated VAE service addresses used by DiT instances."); + +DEFINE_int32(dit_vae_request_timeout_ms, + 600000, + "Timeout for DiT to VAE decode requests in milliseconds."); + +DEFINE_int32(dit_vae_health_check_timeout_ms, + 1000, + "Timeout for DiT to VAE health checks in milliseconds."); + +DEFINE_int32(dit_vae_health_check_interval_ms, + 1000, + "Minimum interval between health checks for an unavailable VAE " + "worker in milliseconds."); + +DEFINE_int32(dit_worker_port, + 0, + "Base worker RPC port for DiT instances; 0 selects random " + "available ports. Global rank is added when nonzero."); + DEFINE_bool(dit_sp_communication_overlap, true, "Communication & Computation overlap for sequence parallel"); @@ -131,6 +157,12 @@ void DiTConfig::from_flags() { XLLM_CONFIG_ASSIGN_FROM_FLAG(dit_cache_end_steps); XLLM_CONFIG_ASSIGN_FROM_FLAG(dit_cache_start_blocks); XLLM_CONFIG_ASSIGN_FROM_FLAG(dit_cache_end_blocks); + XLLM_CONFIG_ASSIGN_FROM_FLAG(dit_instance_role); + XLLM_CONFIG_ASSIGN_FROM_FLAG(dit_vae_service_addresses); + XLLM_CONFIG_ASSIGN_FROM_FLAG(dit_vae_request_timeout_ms); + XLLM_CONFIG_ASSIGN_FROM_FLAG(dit_vae_health_check_timeout_ms); + XLLM_CONFIG_ASSIGN_FROM_FLAG(dit_vae_health_check_interval_ms); + XLLM_CONFIG_ASSIGN_FROM_FLAG(dit_worker_port); XLLM_CONFIG_ASSIGN_FROM_FLAG(dit_sp_communication_overlap); XLLM_CONFIG_ASSIGN_FROM_FLAG(dit_debug_print); XLLM_CONFIG_ASSIGN_FROM_FLAG(dit_laser_attention_enabled); @@ -157,6 +189,12 @@ void DiTConfig::from_json(const JsonReader& json) { XLLM_CONFIG_ASSIGN_FROM_JSON(dit_cache_end_steps); XLLM_CONFIG_ASSIGN_FROM_JSON(dit_cache_start_blocks); XLLM_CONFIG_ASSIGN_FROM_JSON(dit_cache_end_blocks); + XLLM_CONFIG_ASSIGN_FROM_JSON(dit_instance_role); + XLLM_CONFIG_ASSIGN_FROM_JSON(dit_vae_service_addresses); + XLLM_CONFIG_ASSIGN_FROM_JSON(dit_vae_request_timeout_ms); + XLLM_CONFIG_ASSIGN_FROM_JSON(dit_vae_health_check_timeout_ms); + XLLM_CONFIG_ASSIGN_FROM_JSON(dit_vae_health_check_interval_ms); + XLLM_CONFIG_ASSIGN_FROM_JSON(dit_worker_port); XLLM_CONFIG_ASSIGN_FROM_JSON(dit_sp_communication_overlap); XLLM_CONFIG_ASSIGN_FROM_JSON(dit_debug_print); XLLM_CONFIG_ASSIGN_FROM_JSON(dit_laser_attention_enabled); @@ -194,6 +232,18 @@ void DiTConfig::append_config_json(nlohmann::ordered_json& config_json) const { config_json, default_config, dit_cache_start_blocks); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( config_json, default_config, dit_cache_end_blocks); + APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( + config_json, default_config, dit_instance_role); + APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( + config_json, default_config, dit_vae_service_addresses); + APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( + config_json, default_config, dit_vae_request_timeout_ms); + APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( + config_json, default_config, dit_vae_health_check_timeout_ms); + APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( + config_json, default_config, dit_vae_health_check_interval_ms); + APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( + config_json, default_config, dit_worker_port); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( config_json, default_config, dit_sp_communication_overlap); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( @@ -232,6 +282,22 @@ void DiTConfig::initialize() { if (const auto& json_config = config::get_parsed_json_config()) { from_json(*json_config); } + CHECK(dit_instance_role() == "all" || dit_instance_role() == "dit" || + dit_instance_role() == "vae") + << "Unsupported dit_instance_role: " << dit_instance_role(); + CHECK(dit_vae_request_timeout_ms() == -1 || dit_vae_request_timeout_ms() > 0) + << "dit_vae_request_timeout_ms must be -1 or positive, got " + << dit_vae_request_timeout_ms(); + CHECK_GT(dit_vae_health_check_timeout_ms(), 0) + << "dit_vae_health_check_timeout_ms must be positive, got " + << dit_vae_health_check_timeout_ms(); + CHECK_GT(dit_vae_health_check_interval_ms(), 0) + << "dit_vae_health_check_interval_ms must be positive, got " + << dit_vae_health_check_interval_ms(); + CHECK_GE(dit_worker_port(), 0) + << "dit_worker_port must be non-negative, got " << dit_worker_port(); + CHECK_LE(dit_worker_port(), 65535) + << "dit_worker_port exceeds the valid port range: " << dit_worker_port(); } } // namespace xllm diff --git a/xllm/core/framework/config/dit_config.h b/xllm/core/framework/config/dit_config.h index 6eaa3209f1..ea1c95ec71 100644 --- a/xllm/core/framework/config/dit_config.h +++ b/xllm/core/framework/config/dit_config.h @@ -51,6 +51,12 @@ class DiTConfig final { "dit_cache_end_steps", "dit_cache_start_blocks", "dit_cache_end_blocks", + "dit_instance_role", + "dit_vae_service_addresses", + "dit_vae_request_timeout_ms", + "dit_vae_health_check_timeout_ms", + "dit_vae_health_check_interval_ms", + "dit_worker_port", "dit_sp_communication_overlap", "dit_debug_print", "dit_laser_attention_enabled", @@ -87,6 +93,18 @@ class DiTConfig final { PROPERTY(int64_t, dit_cache_end_blocks) = 5; + PROPERTY(std::string, dit_instance_role) = "all"; + + PROPERTY(std::string, dit_vae_service_addresses); + + PROPERTY(int32_t, dit_vae_request_timeout_ms) = 600000; + + PROPERTY(int32_t, dit_vae_health_check_timeout_ms) = 1000; + + PROPERTY(int32_t, dit_vae_health_check_interval_ms) = 1000; + + PROPERTY(int32_t, dit_worker_port) = 0; + PROPERTY(bool, dit_sp_communication_overlap) = true; PROPERTY(bool, dit_debug_print) = false; diff --git a/xllm/core/framework/request/dit_request.cpp b/xllm/core/framework/request/dit_request.cpp index 904c476884..5ec42d44ce 100644 --- a/xllm/core/framework/request/dit_request.cpp +++ b/xllm/core/framework/request/dit_request.cpp @@ -24,6 +24,7 @@ limitations under the License. #include #include #include +#include #include #include "api_service/call.h" @@ -127,6 +128,8 @@ void DiTRequest::handle_forward_output(torch::Tensor output) { output_.tensors = torch::chunk(output, count); } +void DiTRequest::handle_error(Status status) { status_ = std::move(status); } + void DiTRequest::handle_forward_text_output(const std::string& text) { output_.text_output.push_back(text); } @@ -135,10 +138,14 @@ const DiTRequestOutput DiTRequest::generate_output() { DiTRequestOutput output; output.request_id = request_id_; output.service_request_id = service_request_id_; - output.status = Status(StatusCode::OK); + output.status = status_.value_or(Status(StatusCode::OK)); output.finished = finished(); output.cancelled = false; + if (!output.status->ok()) { + return output; + } + // Text diffusion models (e.g., Cola-DLM) produce text output directly. if (!output_.text_output.empty()) { const auto& gen_params = state_.generation_params(); diff --git a/xllm/core/framework/request/dit_request.h b/xllm/core/framework/request/dit_request.h index 5d5a43917b..26fddc739c 100644 --- a/xllm/core/framework/request/dit_request.h +++ b/xllm/core/framework/request/dit_request.h @@ -20,6 +20,7 @@ limitations under the License. #include #include +#include #include #include @@ -45,6 +46,8 @@ class DiTRequest : public RequestBase { void handle_forward_output(torch::Tensor output); + void handle_error(Status status); + void handle_forward_text_output(const std::string& text); const DiTRequestOutput generate_output(); @@ -56,6 +59,7 @@ class DiTRequest : public RequestBase { private: DiTRequestState state_; DiTForwardOutput output_; + std::optional status_; }; } // namespace xllm diff --git a/xllm/core/scheduler/dit_scheduler.cpp b/xllm/core/scheduler/dit_scheduler.cpp index 1ae2994d8a..5c76017164 100644 --- a/xllm/core/scheduler/dit_scheduler.cpp +++ b/xllm/core/scheduler/dit_scheduler.cpp @@ -129,7 +129,13 @@ void DiTAsyncResponseProcessor::process_completed_request( void DiTAsyncResponseProcessor::process_failed_request( std::shared_ptr request, - Status status) {} + Status status) { + response_threadpool_.schedule( + [request = std::move(request), status = std::move(status)]() mutable { + request->handle_error(std::move(status)); + request->state().output_func()(request->generate_output()); + }); +} DiTDynamicBatchScheduler::DiTDynamicBatchScheduler(Engine* engine, const Options& options) diff --git a/xllm/models/dit/pipelines/pipeline_flux.h b/xllm/models/dit/pipelines/pipeline_flux.h index 60d53d8e45..e0ae70ba42 100644 --- a/xllm/models/dit/pipelines/pipeline_flux.h +++ b/xllm/models/dit/pipelines/pipeline_flux.h @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #pragma once +#include "core/framework/config/dit_config.h" #include "models/dit/pipelines/pipeline_flux_base.h" #include "models/dit/transformers/transformer_flux.h" // pipeline_flux compatible with huggingface weights @@ -60,6 +61,24 @@ class FluxPipelineImpl : public FluxPipelineBaseImpl { } DiTForwardOutput forward(const DiTForwardInput& input) { + const std::string& instance_role = + DiTConfig::get_instance().dit_instance_role(); + CHECK(instance_role == "all" || instance_role == "dit" || + instance_role == "vae") + << "Unsupported dit_instance_role: " << instance_role; + + if (instance_role == "vae") { + CHECK(input.latents.defined()) + << "VAE instance requires latent input from a DiT instance."; + torch::Tensor latents = input.latents.to(options_.device()); + torch::Tensor image = decode_latents(latents, + input.generation_params.height, + input.generation_params.width); + DiTForwardOutput out; + out.tensors = torch::chunk(image, input.batch_size); + return out; + } + const DiTGenerationParams& generation_params = input.generation_params; int64_t seed = generation_params.seed > 0 ? generation_params.seed : 42; std::optional> prompts = @@ -131,16 +150,24 @@ class FluxPipelineImpl : public FluxPipelineBaseImpl { auto tokenizer_2_loader = loader->take_component_loader("tokenizer_2"); LOG(INFO) << "Flux model components loaded, start to load weights to sub models"; - transformer_->load_model(std::move(transformer_loader)); - transformer_->to(options_.device()); - vae_->load_model(std::move(vae_loader)); - vae_->to(options_.device()); - t5_->load_model(std::move(t5_loader)); - t5_->to(options_.device()); - clip_text_model_->load_model(std::move(clip_loader)); - clip_text_model_->to(options_.device()); - tokenizer_ = tokenizer_loader->tokenizer(); - tokenizer_2_ = tokenizer_2_loader->tokenizer(); + const std::string& instance_role = + DiTConfig::get_instance().dit_instance_role(); + if (instance_role != "vae") { + transformer_->load_model(std::move(transformer_loader)); + transformer_->to(options_.device()); + t5_->load_model(std::move(t5_loader)); + t5_->to(options_.device()); + clip_text_model_->load_model(std::move(clip_loader)); + clip_text_model_->to(options_.device()); + } + if (instance_role != "dit") { + vae_->load_model(std::move(vae_loader)); + vae_->to(options_.device()); + } + if (instance_role != "vae") { + tokenizer_ = tokenizer_loader->tokenizer(); + tokenizer_2_ = tokenizer_2_loader->tokenizer(); + } } private: @@ -312,16 +339,23 @@ class FluxPipelineImpl : public FluxPipelineBaseImpl { prepared_latents = prepared_latents.to(latents.value().dtype()); } } - torch::Tensor image; - // Unpack latents + const std::string& instance_role = + DiTConfig::get_instance().dit_instance_role(); + if (instance_role == "dit") { + return prepared_latents; + } + return decode_latents(prepared_latents, height, width); + } + + torch::Tensor decode_latents(const torch::Tensor& latents, + int64_t height, + int64_t width) { torch::Tensor unpacked_latents = - unpack_latents(prepared_latents, height, width, vae_scale_factor_); + unpack_latents(latents, height, width, vae_scale_factor_); unpacked_latents = (unpacked_latents / vae_scaling_factor_) + vae_shift_factor_; unpacked_latents = unpacked_latents.to(options_.dtype()); - image = vae_->decode(unpacked_latents); - image = vae_image_processor_->postprocess(image); - return image; + return vae_image_processor_->postprocess(vae_->decode(unpacked_latents)); } private: