20231088/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

pass ignore_eos parameter to all benchmark_serving calls (#9349)

Browse Source
This commit is contained in:
Grace Ho 2024-10-15 13:30:44 -07:00 committed by GitHub
parent e9d517f276
commit 5d264f4ab8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 18 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
benchmarks/benchmark_serving.py
View File

@ -431,16 +431,15 @@ async def benchmark(
if profile: if profile:
print("Starting profiler...") print("Starting profiler...")
profile_input = RequestFuncInput( profile_input = RequestFuncInput(model=model_id,
model=model_id, prompt=test_prompt,
prompt=test_prompt, api_url=base_url + "/start_profile",
api_url=base_url + "/start_profile", prompt_len=test_prompt_len,
prompt_len=test_prompt_len, output_len=test_output_len,
output_len=test_output_len, logprobs=logprobs,
logprobs=logprobs, best_of=best_of,
best_of=best_of, multi_modal_content=test_mm_content,
multi_modal_content=test_mm_content, ignore_eos=ignore_eos)
)
profile_output = await request_func(request_func_input=profile_input) profile_output = await request_func(request_func_input=profile_input)
if profile_output.success: if profile_output.success:
print("Profiler started") print("Profiler started")
@ -453,16 +452,15 @@ async def benchmark(
tasks: List[asyncio.Task] = [] tasks: List[asyncio.Task] = []
async for request in get_request(input_requests, request_rate): async for request in get_request(input_requests, request_rate):
prompt, prompt_len, output_len, mm_content = request prompt, prompt_len, output_len, mm_content = request
request_func_input = RequestFuncInput( request_func_input = RequestFuncInput(model=model_id,
model=model_id, prompt=prompt,
prompt=prompt, api_url=api_url,
api_url=api_url, prompt_len=prompt_len,
prompt_len=prompt_len, output_len=output_len,
output_len=output_len, logprobs=logprobs,
logprobs=logprobs, best_of=best_of,
best_of=best_of, multi_modal_content=mm_content,
multi_modal_content=mm_content, ignore_eos=ignore_eos)
)
tasks.append( tasks.append(
asyncio.create_task( asyncio.create_task(
request_func(request_func_input=request_func_input, request_func(request_func_input=request_func_input,