20231088/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Bugfix] Fix ITL recording in serving benchmark (#7372)

Browse Source
This commit is contained in:
Roger Wang 2024-08-09 10:00:00 -07:00 committed by GitHub
parent 249b88228d
commit 70d268a399
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
benchmarks/backend_request_func.py
View File

@ -276,8 +276,9 @@ async def async_request_openai_completions(
output.ttft = ttft output.ttft = ttft
# Decoding phase # Decoding phase
output.itl.append(timestamp - else:
most_recent_timestamp) output.itl.append(timestamp -
most_recent_timestamp)
most_recent_timestamp = timestamp most_recent_timestamp = timestamp
generated_text += data["choices"][0]["text"] generated_text += data["choices"][0]["text"]