104 lines
3.6 KiB
Python
104 lines
3.6 KiB
Python
from typing import List, Optional, Tuple
|
|
|
|
import pytest
|
|
from transformers import AutoModelForVision2Seq, AutoTokenizer
|
|
|
|
from vllm.multimodal.utils import rescale_image_size
|
|
from vllm.sequence import SampleLogprobs
|
|
|
|
from ..conftest import IMAGE_ASSETS
|
|
from .utils import check_logprobs_close
|
|
|
|
pytestmark = pytest.mark.vlm
|
|
|
|
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
|
|
"stop_sign":
|
|
"Question: What's the content of the image? Answer:",
|
|
"cherry_blossom":
|
|
"Question: What is the season? Answer:",
|
|
})
|
|
|
|
|
|
def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
|
|
Optional[SampleLogprobs]],
|
|
model: str):
|
|
"""Sanitize vllm output to be comparable with hf output."""
|
|
_, output_str, out_logprobs = vllm_output
|
|
|
|
hf_output_str = output_str + "\n"
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained(model)
|
|
hf_output_ids = tokenizer.encode(hf_output_str)
|
|
assert hf_output_ids[0] == tokenizer.bos_token_id
|
|
hf_output_ids = hf_output_ids[1:]
|
|
|
|
return hf_output_ids, hf_output_str, out_logprobs
|
|
|
|
|
|
@pytest.mark.parametrize("model", ["Salesforce/blip2-opt-2.7b"])
|
|
@pytest.mark.parametrize(
|
|
"size_factors",
|
|
[
|
|
# No image
|
|
[],
|
|
# Single-scale
|
|
[1.0],
|
|
# Single-scale, batched
|
|
[1.0, 1.0, 1.0],
|
|
# Multi-scale
|
|
[0.25, 0.5, 1.0],
|
|
],
|
|
)
|
|
@pytest.mark.parametrize("dtype", ["half"])
|
|
@pytest.mark.parametrize("max_tokens", [128])
|
|
@pytest.mark.parametrize("num_logprobs", [5])
|
|
def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
|
|
dtype: str, max_tokens: int, num_logprobs: int) -> None:
|
|
"""Inference result should be the same between hf and vllm.
|
|
|
|
All the image fixtures for the test is under tests/images.
|
|
For huggingface runner, we provide the PIL images as input.
|
|
For vllm runner, we provide MultiModalData objects and corresponding
|
|
MultiModalConfig as input.
|
|
Note, the text input is also adjusted to abide by vllm contract.
|
|
The text output is sanitized to be able to compare with hf.
|
|
"""
|
|
images = [asset.pil_image for asset in image_assets]
|
|
|
|
inputs_per_image = [(
|
|
[prompt for _ in size_factors],
|
|
[rescale_image_size(image, factor) for factor in size_factors],
|
|
) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
|
|
|
|
# max_model_len should be greater than image_feature_size
|
|
with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
|
|
vllm_outputs_per_image = [
|
|
vllm_model.generate_greedy_logprobs(prompts,
|
|
max_tokens,
|
|
num_logprobs=num_logprobs,
|
|
images=images)
|
|
for prompts, images in inputs_per_image
|
|
]
|
|
|
|
with hf_runner(model, dtype=dtype,
|
|
auto_cls=AutoModelForVision2Seq) as hf_model:
|
|
hf_outputs_per_image = [
|
|
hf_model.generate_greedy_logprobs_limit(prompts,
|
|
max_tokens,
|
|
num_logprobs=num_logprobs,
|
|
images=images)
|
|
for prompts, images in inputs_per_image
|
|
]
|
|
|
|
for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
|
|
vllm_outputs_per_image):
|
|
check_logprobs_close(
|
|
outputs_0_lst=hf_outputs,
|
|
outputs_1_lst=[
|
|
vllm_to_hf_output(vllm_output, model)
|
|
for vllm_output in vllm_outputs
|
|
],
|
|
name_0="hf",
|
|
name_1="vllm",
|
|
)
|