""" This example shows how to use vLLM for running offline inference with the correct prompt format on vision language models. For most models, the prompt format should follow corresponding examples on HuggingFace model repository. """ from transformers import AutoTokenizer from vllm import LLM, SamplingParams from vllm.assets.image import ImageAsset from vllm.utils import FlexibleArgumentParser # Input image and question image = ImageAsset("cherry_blossom").pil_image.convert("RGB") question = "What is the content of this image?" # LLaVA-1.5 def run_llava(question): prompt = f"USER: \n{question}\nASSISTANT:" llm = LLM(model="llava-hf/llava-1.5-7b-hf") stop_token_ids = None return llm, prompt, stop_token_ids # LLaVA-1.6/LLaVA-NeXT def run_llava_next(question): prompt = f"[INST] \n{question} [/INST]" llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf") stop_token_ids = None return llm, prompt, stop_token_ids # Fuyu def run_fuyu(question): prompt = f"{question}\n" llm = LLM(model="adept/fuyu-8b") stop_token_ids = None return llm, prompt, stop_token_ids # Phi-3-Vision def run_phi3v(question): prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n" # noqa: E501 # Note: The default setting of max_num_seqs (256) and # max_model_len (128k) for this model may cause OOM. # You may lower either to run this example on lower-end GPUs. # In this example, we override max_num_seqs to 5 while # keeping the original context length of 128k. llm = LLM( model="microsoft/Phi-3-vision-128k-instruct", trust_remote_code=True, max_num_seqs=5, ) stop_token_ids = None return llm, prompt, stop_token_ids # PaliGemma def run_paligemma(question): # PaliGemma has special prompt format for VQA prompt = "caption en" llm = LLM(model="google/paligemma-3b-mix-224") stop_token_ids = None return llm, prompt, stop_token_ids # Chameleon def run_chameleon(question): prompt = f"{question}" llm = LLM(model="facebook/chameleon-7b") stop_token_ids = None return llm, prompt, stop_token_ids # MiniCPM-V def run_minicpmv(question): # 2.0 # The official repo doesn't work yet, so we need to use a fork for now # For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 # noqa # model_name = "HwwwH/MiniCPM-V-2" # 2.5 # model_name = "openbmb/MiniCPM-Llama3-V-2_5" #2.6 model_name = "openbmb/MiniCPM-V-2_6" tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) llm = LLM( model=model_name, trust_remote_code=True, ) # NOTE The stop_token_ids are different for various versions of MiniCPM-V # 2.0 # stop_token_ids = [tokenizer.eos_id] # 2.5 # stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id] # 2.6 stop_tokens = ['<|im_end|>', '<|endoftext|>'] stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens] messages = [{ 'role': 'user', 'content': f'(./)\n{question}' }] prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) return llm, prompt, stop_token_ids # InternVL def run_internvl(question): model_name = "OpenGVLab/InternVL2-2B" llm = LLM( model=model_name, trust_remote_code=True, max_num_seqs=5, ) tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) messages = [{'role': 'user', 'content': f"\n{question}"}] prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) # Stop tokens for InternVL # models variants may have different stop tokens # please refer to the model card for the correct "stop words": # https://huggingface.co/OpenGVLab/InternVL2-2B#service stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"] stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens] return llm, prompt, stop_token_ids # BLIP-2 def run_blip2(question): # BLIP-2 prompt format is inaccurate on HuggingFace model repository. # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa prompt = f"Question: {question} Answer:" llm = LLM(model="Salesforce/blip2-opt-2.7b") stop_token_ids = None return llm, prompt, stop_token_ids # Qwen def run_qwen_vl(question): llm = LLM( model="Qwen/Qwen-VL", trust_remote_code=True, max_num_seqs=5, ) prompt = f"{question}Picture 1: \n" stop_token_ids = None return llm, prompt, stop_token_ids model_example_map = { "llava": run_llava, "llava-next": run_llava_next, "fuyu": run_fuyu, "phi3_v": run_phi3v, "paligemma": run_paligemma, "chameleon": run_chameleon, "minicpmv": run_minicpmv, "blip-2": run_blip2, "internvl_chat": run_internvl, "qwen_vl": run_qwen_vl, } def main(args): model = args.model_type if model not in model_example_map: raise ValueError(f"Model type {model} is not supported.") llm, prompt, stop_token_ids = model_example_map[model](question) # We set temperature to 0.2 so that outputs can be different # even when all prompts are identical when running batch inference. sampling_params = SamplingParams(temperature=0.2, max_tokens=64, stop_token_ids=stop_token_ids) assert args.num_prompts > 0 if args.num_prompts == 1: # Single inference inputs = { "prompt": prompt, "multi_modal_data": { "image": image }, } else: # Batch inference inputs = [{ "prompt": prompt, "multi_modal_data": { "image": image }, } for _ in range(args.num_prompts)] outputs = llm.generate(inputs, sampling_params=sampling_params) for o in outputs: generated_text = o.outputs[0].text print(generated_text) if __name__ == "__main__": parser = FlexibleArgumentParser( description='Demo on using vLLM for offline inference with ' 'vision language models') parser.add_argument('--model-type', '-m', type=str, default="llava", choices=model_example_map.keys(), help='Huggingface "model_type".') parser.add_argument('--num-prompts', type=int, default=1, help='Number of prompts to run.') args = parser.parse_args() main(args)