20231088/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Bugfix] Fix compressed_tensors_moe bad config.strategy (#9677)

Browse Source
This commit is contained in:
Michael Goin 2024-10-25 15:40:33 -04:00 committed by GitHub
parent 9645b9f646
commit ca0d92227e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
View File

@ -245,7 +245,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
config = self.quant_config.target_scheme_map["Linear"].get("weights") config = self.quant_config.target_scheme_map["Linear"].get("weights")
self.num_bits = config.num_bits self.num_bits = config.num_bits
self.packed_factor = 32 // config.num_bits self.packed_factor = 32 // config.num_bits
self.strategy = config.strategy.value self.strategy = config.strategy
self.group_size = config.group_size self.group_size = config.group_size
assert config.symmetric, ( assert config.symmetric, (
"Only symmetric quantization is supported for MoE") "Only symmetric quantization is supported for MoE")