From a88bb9b032d75aad74b2e1bd3d97b8e8a24e8b9d Mon Sep 17 00:00:00 2001 From: AnyISalIn Date: Thu, 2 May 2024 00:11:03 +0800 Subject: [PATCH] [Bugfix] Fix the fp8 kv_cache check error that occurs when failing to obtain the CUDA version. (#4173) Signed-off-by: AnyISalIn --- vllm/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm/config.py b/vllm/config.py index a5512c65..db4398ad 100644 --- a/vllm/config.py +++ b/vllm/config.py @@ -353,7 +353,8 @@ class CacheConfig: elif self.cache_dtype == "fp8": if not is_hip(): nvcc_cuda_version = get_nvcc_cuda_version() - if nvcc_cuda_version < Version("11.8"): + if nvcc_cuda_version is not None \ + and nvcc_cuda_version < Version("11.8"): raise ValueError( "FP8 is not supported when cuda version is" "lower than 11.8.")