20231088/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Bugfix] Use LoadFormat values for vllm serve --load-format (#7784)

Browse Source
This commit is contained in:
Michael Goin 2024-08-22 14:37:08 -04:00 committed by GitHub
parent 57792ed469
commit 15310b5101
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
vllm/engine/arg_utils.py
View File

@ -9,8 +9,8 @@ import torch
import vllm.envs as envs import vllm.envs as envs
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
EngineConfig, LoadConfig, LoRAConfig, ModelConfig, EngineConfig, LoadConfig, LoadFormat, LoRAConfig,
ObservabilityConfig, ParallelConfig, ModelConfig, ObservabilityConfig, ParallelConfig,
PromptAdapterConfig, SchedulerConfig, PromptAdapterConfig, SchedulerConfig,
SpeculativeConfig, TokenizerPoolConfig) SpeculativeConfig, TokenizerPoolConfig)
from vllm.executor.executor_base import ExecutorBase from vllm.executor.executor_base import ExecutorBase
@ -214,10 +214,7 @@ class EngineArgs:
'--load-format', '--load-format',
type=str, type=str,
default=EngineArgs.load_format, default=EngineArgs.load_format,
choices=[ choices=[f.value for f in LoadFormat],
'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer',
'bitsandbytes'
],
help='The format of the model weights to load.\n\n' help='The format of the model weights to load.\n\n'
'* "auto" will try to load the weights in the safetensors format ' '* "auto" will try to load the weights in the safetensors format '
'and fall back to the pytorch bin format if safetensors format ' 'and fall back to the pytorch bin format if safetensors format '