forked from mlcommons/inference
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmlperf.conf
More file actions
199 lines (175 loc) · 8.23 KB
/
Copy pathmlperf.conf
File metadata and controls
199 lines (175 loc) · 8.23 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
# The format of this config file is 'key = value'.
# The key has the format 'model.scenario.key'. Value is mostly int64_t.
# Model maybe '*' as wildcard. In that case the value applies to all models.
# All times are in milli seconds
# Set performance_sample_count for each model.
# User can optionally set this to higher values in user.conf.
resnet50.*.performance_sample_count_override = 1024
ssd-mobilenet.*.performance_sample_count_override = 256
retinanet.*.performance_sample_count_override = 64
bert.*.performance_sample_count_override = 10833
dlrm.*.performance_sample_count_override = 204800
dlrm-v2.*.performance_sample_count_override = 204800
rnnt.*.performance_sample_count_override = 2513
gptj.*.performance_sample_count_override = 13368
mixtral-8x7b.*.performance_sample_count_override = 15000
llama2-70b.*.performance_sample_count_override = 24576
llama2-70b-interactive.*.performance_sample_count_override = 24576
llama3_1-405b.*.performance_sample_count_override = 8313
llama3_1-405b-interactive.*.performance_sample_count_override = 8313
llama3_1-8b.*.performance_sample_count_override = 13368
llama3_1-8b-edge.*.performance_sample_count_override = 5000
llama3_1-8b-interactive.*.performance_sample_count_override = 13368
stable-diffusion-xl.*.performance_sample_count_override = 5000
wan-2.2-t2v-a14b.*.performance_sample_count_override = 248
rgat.*.performance_sample_count_override = 788379
pointpainting.*.performance_sample_count_override = 1024
deepseek-r1.*.performance_sample_count_override = 4388
deepseek-r1-interactive.*.performance_sample_count_override = 4388
whisper.*.performance_sample_count_override = 1633
qwen3-vl-235b-a22b.*.performance_sample_count_override = 48289
yolo.*.performance_sample_count_override = 64
# set to 0 to let entire sample set to be performance sample
3d-unet.*.performance_sample_count_override = 0
gpt-oss-120b.*.performance_sample_count_override = 6396
gpt-oss-120b.*.accuracy_sample_count_override = 4395
gpt-oss-120b-interactive.*.performance_sample_count_override = 6396
gpt-oss-120b-interactive.*.accuracy_sample_count_override = 4395
# Set seeds.
*.*.qsl_rng_seed = 2085463073848966840
*.*.sample_index_rng_seed = 2747215439041700203
*.*.schedule_rng_seed = 16159082839903944936
# Set seeds for TEST_05 (not needed from v5.0 onwards)
*.*.test05_qsl_rng_seed = 7975553102935885558
*.*.test05_sample_index_rng_seed = 11403566307062068064
*.*.test05_schedule_rng_seed = 15816800565822761601
*.SingleStream.target_latency_percentile = 90
pointpainting.SingleStream.target_latency_percentile = 99.9
*.SingleStream.min_duration = 600000
*.MultiStream.target_latency_percentile = 99
*.MultiStream.samples_per_query = 8
*.MultiStream.min_duration = 600000
*.MultiStream.min_query_count = 662
retinanet.MultiStream.target_latency = 528
# 3D-UNet uses equal issue mode because it has non-uniform inputs
3d-unet.*.sample_concatenate_permutation = 1
# R-GAT uses equal issue mode because it may have non-uniform inputs
rgat.*.sample_concatenate_permutation = 1
# LLM benchmarks have non-uniform inputs and outputs, and use equal issue mode for all latency scenario
gptj.*.sample_concatenate_permutation = 1
llama2-70b.*.sample_concatenate_permutation = 1
llama2-70b-interactive.*.sample_concatenate_permutation = 1
mixtral-8x7b.*.sample_concatenate_permutation = 1
llama3_1-405b.*.sample_concatenate_permutation = 1
llama3_1-405b-interactive.*.sample_concatenate_permutation = 1
llama3_1-8b.*.sample_concatenate_permutation = 1
llama3_1-8b-edge.*.sample_concatenate_permutation = 1
llama3_1-8b-interactive.*.sample_concatenate_permutation = 1
deepseek-r1.*.sample_concatenate_permutation = 1
deepseek-r1-interactive.*.sample_concatenate_permutation = 1
whisper.*.sample_concatenate_permutation = 1
qwen3-vl-235b-a22b.*.sample_concatenate_permutation = 1
gpt-oss-120b.*.sample_concatenate_permutation = 1
gpt-oss-120b-interactive.*.sample_concatenate_permutation = 1
*.Server.target_latency = 10
*.Server.target_latency_percentile = 99
*.Server.target_duration = 0
*.Server.min_duration = 600000
resnet50.Server.target_latency = 15
retinanet.Server.target_latency = 100
bert.Server.target_latency = 130
dlrm.Server.target_latency = 60
dlrm-v2.Server.target_latency = 60
dlrm-v3.Server.target_latency = 80
rnnt.Server.target_latency = 1000
gptj.Server.target_latency = 20000
stable-diffusion-xl.Server.target_latency = 20000
# Benchmarks that measure token latencies
llama2-70b.*.use_token_latencies = 1
llama2-70b-interactive.*.use_token_latencies = 1
mixtral-8x7b.*.use_token_latencies = 1
llama3_1-405b.*.use_token_latencies = 1
llama3_1-405b-interactive.*.use_token_latencies = 1
llama3_1-8b.*.use_token_latencies = 1
llama3_1-8b-edge.*.use_token_latencies = 1
llama3_1-8b-interactive.*.use_token_latencies = 1
deepseek-r1.*.use_token_latencies = 1
deepseek-r1-interactive.*.use_token_latencies = 1
whisper.*.use_token_latencies = 1
gpt-oss-120b.*.use_token_latencies = 1
gpt-oss-120b-interactive.*.use_token_latencies = 1
# For the VLM benchmark, the model response is relatively short, therefore we track
# end-to-end latency instead of token latencies.
qwen3-vl-235b-a22b.*.use_token_latencies = 0
# gptj benchmark infers token latencies
gptj.*.infer_token_latencies = 1
gptj.*.token_latency_scaling_factor = 69
# Only ttft and tpot are tracked for the llama2-70b, mixtral-8x7B, llama3_1-405b & gpt-oss-120b benchmark therefore target_latency = 0
llama2-70b.Server.target_latency = 0
llama2-70b.Server.ttft_latency = 2000
llama2-70b.Server.tpot_latency = 200
# Target Latencies for interactive setting
llama2-70b-interactive.Server.target_latency = 0
llama2-70b-interactive.Server.ttft_latency = 450
llama2-70b-interactive.Server.tpot_latency = 40
mixtral-8x7b.Server.target_latency = 0
mixtral-8x7b.Server.ttft_latency = 2000
mixtral-8x7b.Server.tpot_latency = 200
llama3_1-405b.Server.target_latency = 0
llama3_1-405b.Server.ttft_latency = 6000
llama3_1-405b.Server.tpot_latency = 175
# Target Latencies for interactive setting
llama3_1-405b-interactive.Server.target_latency = 0
llama3_1-405b-interactive.Server.ttft_latency = 4500
llama3_1-405b-interactive.Server.tpot_latency = 80
llama3_1-8b.Server.target_latency = 0
llama3_1-8b.Server.ttft_latency = 2000
llama3_1-8b.Server.tpot_latency = 100
# Target Latencies for interactive setting
llama3_1-8b-interactive.Server.target_latency = 0
llama3_1-8b-interactive.Server.ttft_latency = 500
llama3_1-8b-interactive.Server.tpot_latency = 30
deepseek-r1.Server.target_latency = 0
deepseek-r1.Server.ttft_latency = 2000
deepseek-r1.Server.tpot_latency = 80
# Target Latencies for interactive setting
deepseek-r1-interactive.Server.target_latency = 0
deepseek-r1-interactive.Server.ttft_latency = 1500
deepseek-r1-interactive.Server.tpot_latency = 15
gpt-oss-120b.Server.target_latency = 0
gpt-oss-120b.Server.ttft_latency = 3000
gpt-oss-120b.Server.tpot_latency = 80
# Target Latencies for interactive setting
gpt-oss-120b-interactive.Server.target_latency = 0
gpt-oss-120b-interactive.Server.ttft_latency = 2000
gpt-oss-120b-interactive.Server.tpot_latency = 15
qwen3-vl-235b-a22b.Server.target_latency = 12000
*.Offline.target_latency_percentile = 90
*.Offline.min_duration = 600000
# In Offline scenario, we always have one query. But LoadGen maps this to
# min_sample_count internally in Offline scenario. If the dataset size is larger
# than 24576 we limit the min_query_count to 24576 and otherwise we use
# the dataset size as the limit
resnet50.Offline.min_query_count = 24576
retinanet.Offline.min_query_count = 24576
dlrm-v2.Offline.min_query_count = 24576
bert.Offline.min_query_count = 10833
gptj.Offline.min_query_count = 13368
rnnt.Offline.min_query_count = 2513
3d-unet.Offline.min_query_count = 43
stable-diffusion-xl.Offline.min_query_count = 5000
wan-2.2-t2v-a14b.Offline.min_query_count = 248
llama2-70b.Offline.min_query_count = 24576
llama3_1-405b.Offline.min_query_count = 8313
llama3_1-8b.Offline.min_query_count = 13368
llama3_1-8b-edge.Offline.min_query_count = 5000
mixtral-8x7b.Offline.min_query_count = 15000
rgat.Offline.min_query_count = 788379
deepseek-r1.Offline.min_query_count = 4388
whisper.Offline.min_query_count = 1633
qwen3-vl-235b-a22b.Offline.min_query_count = 48289
# These fields should be defined and overridden by user.conf.
*.SingleStream.target_latency = 10
*.MultiStream.target_latency = 80
*.Server.target_qps = 1.0
*.Offline.target_qps = 1.0