98d6682cd1
Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>
90 lines
2.4 KiB
Python
90 lines
2.4 KiB
Python
"""An example showing how to use vLLM to serve VLMs.
|
||
|
||
Launch the vLLM server with the following command:
|
||
python -m vllm.entrypoints.openai.api_server \
|
||
--model llava-hf/llava-1.5-7b-hf \
|
||
--image-token-id 32000 \
|
||
--image-input-shape 1,3,336,336 \
|
||
--image-feature-size 576 \
|
||
--chat-template template_llava.jinja
|
||
"""
|
||
import base64
|
||
|
||
import requests
|
||
from openai import OpenAI
|
||
|
||
# Modify OpenAI's API key and API base to use vLLM's API server.
|
||
openai_api_key = "EMPTY"
|
||
openai_api_base = "http://localhost:8000/v1"
|
||
|
||
client = OpenAI(
|
||
# defaults to os.environ.get("OPENAI_API_KEY")
|
||
api_key=openai_api_key,
|
||
base_url=openai_api_base,
|
||
)
|
||
|
||
models = client.models.list()
|
||
model = models.data[0].id
|
||
|
||
image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
||
|
||
# Use image url in the payload
|
||
chat_completion_from_url = client.chat.completions.create(
|
||
messages=[{
|
||
"role":
|
||
"user",
|
||
"content": [
|
||
{
|
||
"type": "text",
|
||
"text": "What’s in this image?"
|
||
},
|
||
{
|
||
"type": "image_url",
|
||
"image_url": {
|
||
"url": image_url
|
||
},
|
||
},
|
||
],
|
||
}],
|
||
model=model,
|
||
)
|
||
|
||
result = chat_completion_from_url.choices[0].message.content
|
||
print(f"Chat completion output:{result}")
|
||
|
||
|
||
# Use base64 encoded image in the payload
|
||
def encode_image_base64_from_url(image_url: str) -> str:
|
||
"""Encode an image retrieved from a remote url to base64 format."""
|
||
|
||
with requests.get(image_url) as response:
|
||
response.raise_for_status()
|
||
result = base64.b64encode(response.content).decode('utf-8')
|
||
|
||
return result
|
||
|
||
|
||
image_base64 = encode_image_base64_from_url(image_url=image_url)
|
||
chat_completion_from_base64 = client.chat.completions.create(
|
||
messages=[{
|
||
"role":
|
||
"user",
|
||
"content": [
|
||
{
|
||
"type": "text",
|
||
"text": "What’s in this image?"
|
||
},
|
||
{
|
||
"type": "image_url",
|
||
"image_url": {
|
||
"url": f"data:image/jpeg;base64,{image_base64}"
|
||
},
|
||
},
|
||
],
|
||
}],
|
||
model=model,
|
||
)
|
||
|
||
result = chat_completion_from_base64.choices[0].message.content
|
||
print(f"Chat completion output:{result}")
|