20231088/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Bugfix] Enable Proper attention_bias Usage in Llama Model Configuration (#3767)

Browse Source 
Co-authored-by: roy <jasonailu87@gmail.com>
This commit is contained in:
Kiran R 2024-04-09 01:12:35 +05:30 committed by GitHub
parent f46864d68d
commit bc0c0192d1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/model_executor/models/llama.py
View File

@ -184,6 +184,10 @@ class LlamaDecoderLayer(nn.Module):
max_position_embeddings = getattr(config, "max_position_embeddings",
8192)
sliding_window = getattr(config, "sliding_window", None)
# Support abacusai/Smaug-72B-v0.1 with attention_bias
# Support internlm/internlm-7b with bias
attention_bias = getattr(config, "attention_bias", False) or getattr(
config, "bias", False)
self.self_attn = LlamaAttention(
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
@ -193,7 +197,7 @@ class LlamaDecoderLayer(nn.Module):
rope_scaling=rope_scaling,
max_position_embeddings=max_position_embeddings,
linear_method=linear_method,
bias=getattr(config, "bias", False),
bias=attention_bias,
sliding_window=sliding_window,
)
self.mlp = LlamaMLP(