20231088/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

config check sleep mode support oot platforms (#16562)

Browse Source
This commit is contained in:
Shuqiao Li 2025-04-15 07:31:50 +08:00 committed by GitHub
parent 1eb3c2ed48
commit d2020acac7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/config.py
View File

@ -417,8 +417,10 @@ class ModelConfig:
from vllm.platforms import current_platform from vllm.platforms import current_platform
if self.enable_sleep_mode and not current_platform.is_cuda(): if (self.enable_sleep_mode
raise ValueError("Sleep mode is only supported on CUDA devices.") and not current_platform.is_sleep_mode_available()):
raise ValueError(
"Sleep mode is not supported on current platform.")
hf_config = get_config(self.hf_config_path or self.model, hf_config = get_config(self.hf_config_path or self.model,
trust_remote_code, revision, code_revision, trust_remote_code, revision, code_revision,

3
vllm/platforms/interface.py
View File

@ -148,6 +148,9 @@ class Platform:
"""Stateless version of :func:`torch.cuda.is_available`.""" """Stateless version of :func:`torch.cuda.is_available`."""
return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM) return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)
def is_sleep_mode_available(self) -> bool:
return self._enum == PlatformEnum.CUDA
@classmethod @classmethod
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int, def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
dtype: torch.dtype, kv_cache_dtype: Optional[str], dtype: torch.dtype, kv_cache_dtype: Optional[str],