Skip to content

[Feat] add ttft measure for qwen3vl #128

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ZLkanyo009 wants to merge 1 commit into
base: dev/perf
Choose a base branch
from
Open

[Feat] add ttft measure for qwen3vl #128

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 43 additions & 30 deletions python/sglang/bench_serving.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,23 +315,27 @@ async def async_request_openai_chat_completions(

# image
if request_func_input.image_data:
content_items.extend([
{
"type": "image_url",
"image_url": {"url": img_url},
}
for img_url in request_func_input.image_data
])
content_items.extend(
[
{
"type": "image_url",
"image_url": {"url": img_url},
}
for img_url in request_func_input.image_data
]
)

# audio
if request_func_input.audio_data:
content_items.extend([
{
"type": "audio_url",
"audio_url": {"url": audio_url},
}
for audio_url in request_func_input.audio_data
])
content_items.extend(
[
{
"type": "audio_url",
"audio_url": {"url": audio_url},
}
for audio_url in request_func_input.audio_data
]
)

# text
if request_func_input.prompt:
Expand Down Expand Up @@ -741,8 +745,8 @@ def get_processor(
pretrained_model_name_or_path = get_model(pretrained_model_name_or_path)

# Disable truncation when loading the processor for a quantized model.
# Truncation affects the `create_mm_data_row` function’s ability to correctly
# count input tokens (text/vision) for bench serving, which may lead to
# Truncation affects the `create_mm_data_row` function’s ability to correctly
# count input tokens (text/vision) for bench serving, which may lead to
# inaccurate token statistics.
return AutoProcessor.from_pretrained(
pretrained_model_name_or_path,
Expand Down Expand Up @@ -1327,6 +1331,7 @@ def sample_random_requests(
print(f"#Output tokens: {np.sum(output_lens)}")
return input_requests


# Function from https://github.com/sgl-project/sglang/pull
def encode_wav_data_url(y: np.ndarray, sr: int) -> str:
import io
Expand Down Expand Up @@ -1358,6 +1363,7 @@ def encode_wav_data_url(y: np.ndarray, sr: int) -> str:
b64 = pybase64.b64encode(buf.getvalue()).decode("utf-8")
return f"data:audio/wav;base64,{b64}"


def generate_dummy_audio(duration_s=10, sr=1600):
"""
Generate a float32 audio array of `duration_s` seconds at sampling rate `sr`.
Expand All @@ -1369,10 +1375,11 @@ def generate_dummy_audio(duration_s=10, sr=1600):
y = np.random.uniform(low=-1.0, high=1.0, size=num_samples).astype(np.float32)
return y, sr


def build_no_speical_token_table(tokenizer, special_tokens=None):
"""
build no special token to replace special token randomly generated
for Qwen3-Omni
for Qwen3-Omni
"""
vocab = tokenizer.get_vocab().values()
special_set = set(special_tokens or [])
Expand All @@ -1388,6 +1395,7 @@ def build_no_speical_token_table(tokenizer, special_tokens=None):

return np.array(safe_tokens, dtype=np.int32)


def sample_random_omni_requests(
input_len: int,
output_len: int,
Expand All @@ -1397,15 +1405,15 @@ def sample_random_omni_requests(
processor: AutoProcessor,
tokenizer: PreTrainedTokenizerBase,
return_text: bool = True,
skip_special_tokens: bool = True,
skip_special_tokens: bool = True,
) -> List[DatasetRow]:
"""
Omni request temporary limitation:
----------------------------------
Currently this benchmark only tests the (audio + text) modality.
Audio is generated as a dummy waveform and paired
with the provided random text prompt.
`skip_speical_tokens` default true to avoid common mistakes made
with the provided random text prompt.
`skip_speical_tokens` default true to avoid common mistakes made
in Qwen3-Omni's preprocessing
"""
input_lens = np.random.randint(
Expand Down Expand Up @@ -1441,8 +1449,8 @@ def sample_random_omni_requests(
for j in range(input_lens[i]):
tid = (offsets[i] + i + j) % tokenizer.vocab_size
if (
no_special_tokens is not None
and special_tokens is not None
no_special_tokens is not None
and special_tokens is not None
and tid in special_tokens
):
idx = (tid + j) % len(no_special_tokens)
Expand All @@ -1455,7 +1463,7 @@ def sample_random_omni_requests(
if audio_length and audio_length > 0:
y, sr = generate_dummy_audio(audio_length, 1600)
data_url = encode_wav_data_url(y, sr)
audio_list = [data_url]
audio_list = [data_url]
else:
audio_list = None
input_requests.append(
Expand All @@ -1468,10 +1476,13 @@ def sample_random_omni_requests(
)

print(f"#Input text tokens: {np.sum(input_lens)}")
print(f"#Total input audio lengths: {audio_length*num_prompts if audio_length and audio_length > 0 else 0} seconds")
print(
f"#Total input audio lengths: {audio_length*num_prompts if audio_length and audio_length > 0 else 0} seconds"
)
print(f"#Output tokens: {np.sum(output_lens)}")
return input_requests


def parse_image_resolution(image_resolution: str) -> Tuple[int, int]:
"""Parse image resolution into (width, height).

Expand Down Expand Up @@ -1641,27 +1652,29 @@ def _gen_random_image_data_uri(
and processor.tokenizer.all_special_ids is not None
):
special_tokens = set(processor.tokenizer.all_special_ids)

# Extract special token strings and convert to token_id
special_token_strings = []
special_token_ids = set()

if hasattr(processor.tokenizer, "special_tokens"):
# Extract all special tokens from tokenizer.special_tokens dictionary
for key, value in processor.tokenizer.special_tokens.items():
if isinstance(value, str):
special_token_strings.append(value)
elif isinstance(value, list):
special_token_strings.extend(value)

# Also extract from additional_special_tokens
if hasattr(processor.tokenizer, "additional_special_tokens"):
special_token_strings.extend(processor.tokenizer.additional_special_tokens)

# Convert to token_id
if special_token_strings and hasattr(processor.tokenizer, "convert_tokens_to_ids"):
special_token_ids = set(processor.tokenizer.convert_tokens_to_ids(special_token_strings))

special_token_ids = set(
processor.tokenizer.convert_tokens_to_ids(special_token_strings)
)

# Merge all_special_ids and special_token_ids converted from strings
if special_tokens is not None and special_token_ids:
merged_special_tokens = special_tokens | special_token_ids
Expand Down
27 changes: 27 additions & 0 deletions python/sglang/srt/managers/detokenizer_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,10 @@ def handle_batch_embedding_out(self, recv_obj: BatchEmbeddingOutput):
return recv_obj

def handle_batch_token_id_out(self, recv_obj: BatchTokenIDOutput):
import time

detokenizer_receive_time = time.time()

bs = len(recv_obj.rids)

# Initialize decode status
Expand Down Expand Up @@ -246,6 +250,19 @@ def handle_batch_token_id_out(self, recv_obj: BatchTokenIDOutput):
s.sent_offset = len(output_str)
output_strs.append(incremental_output)

# Record detokenizer send time and create timing lists
detokenizer_send_time = time.time()
detokenizer_receive_times = (
[detokenizer_receive_time] * bs
if hasattr(recv_obj, "output_send_time") and recv_obj.output_send_time
else None
)
detokenizer_send_times = (
[detokenizer_send_time] * bs
if hasattr(recv_obj, "output_send_time") and recv_obj.output_send_time
else None
)

return BatchStrOutput(
rids=recv_obj.rids,
http_worker_ipcs=recv_obj.http_worker_ipcs,
Expand Down Expand Up @@ -274,6 +291,16 @@ def handle_batch_token_id_out(self, recv_obj: BatchTokenIDOutput):
placeholder_tokens_idx=None,
placeholder_tokens_val=None,
token_steps=recv_obj.token_steps,
prefill_preparation_time=recv_obj.prefill_preparation_time,
vit_encoding_time=recv_obj.vit_encoding_time,
llm_prefill_time=recv_obj.llm_prefill_time,
total_prefill_time=recv_obj.total_prefill_time,
scheduler_receive_time=recv_obj.scheduler_receive_time,
model_call_time=recv_obj.model_call_time,
first_token_generated_time=recv_obj.first_token_generated_time,
output_send_time=recv_obj.output_send_time,
detokenizer_receive_time=detokenizer_receive_times,
detokenizer_send_time=detokenizer_send_times,
)

def handle_multimodal_decode_req(self, recv_obj: BatchMultimodalDecodeReq):
Expand Down
26 changes: 26 additions & 0 deletions python/sglang/srt/managers/io_struct.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -857,6 +857,18 @@ class BatchTokenIDOutput(BaseBatchReq):
# The trainer step id. Used to know which step's weights are used for sampling.
token_steps: List[List[int]] = None

# TTFT breakdown timing (in seconds)
prefill_preparation_time: List[float] = None
vit_encoding_time: List[float] = None
llm_prefill_time: List[float] = None
total_prefill_time: List[float] = None

# Scheduler overhead timing (in seconds)
scheduler_receive_time: List[float] = None
model_call_time: List[float] = None
first_token_generated_time: List[float] = None
output_send_time: List[float] = None


@dataclass
class BatchMultimodalDecodeReq(BaseBatchReq):
Expand Down Expand Up @@ -933,6 +945,20 @@ class BatchStrOutput(BaseBatchReq):
# The trainer step id. Used to know which step's weights are used for sampling.
token_steps: List[List[int]] = None

# TTFT breakdown timing (in seconds)
prefill_preparation_time: List[float] = None
vit_encoding_time: List[float] = None
llm_prefill_time: List[float] = None
total_prefill_time: List[float] = None

# Scheduler overhead timing (in seconds)
scheduler_receive_time: List[float] = None
model_call_time: List[float] = None
first_token_generated_time: List[float] = None
output_send_time: List[float] = None
detokenizer_receive_time: List[float] = None
detokenizer_send_time: List[float] = None


@dataclass
class BatchMultimodalOutput(BaseBatchReq):
Expand Down
Loading
Loading