fix the issue that len(tokenizer(prompt)["input_ids"]) > prompt_len (#10524)

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
2024-11-21 19:15:36 +08:00 · 2024-11-21 19:15:36 +08:00 · 8a93a598d9
commit 8a93a598d9
parent 1cfde82ffd
1 changed files with 1 additions and 0 deletions
--- a/benchmarks/backend_request_func.py
+++ b/benchmarks/backend_request_func.py
@ -54,6 +54,7 @@ async def async_request_tgi(
            "do_sample": True,
            "temperature": 0.01,  # TGI does not accept 0.0 temperature.
            "top_p": 0.99,  # TGI does not accept 1.0 top_p.
+            "truncate": request_func_input.prompt_len,
            # TGI does not accept ignore_eos flag.
        }
        payload = {