20231088/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Bugfix] fix output parsing error for trtllm backend (#4137)

Browse Source 
Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
Elinx 2024-04-17 19:07:23 +08:00 committed by GitHub
parent 8438e0569e
commit fe3b5bbc23
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
benchmarks/backend_request_func.py
View File

@ -135,6 +135,7 @@ async def async_request_trt_llm(
"data:") "data:")
data = json.loads(chunk) data = json.loads(chunk)
output.generated_text += data["text_output"]
timestamp = time.perf_counter() timestamp = time.perf_counter()
# First token # First token
if ttft == 0.0: if ttft == 0.0:
@ -149,7 +150,6 @@ async def async_request_trt_llm(
most_recent_timestamp = timestamp most_recent_timestamp = timestamp
output.latency = most_recent_timestamp - st output.latency = most_recent_timestamp - st
output.generated_text = json.loads(data)["text_output"]
output.success = True output.success = True
else: else: