20231088/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Frontend] Enable decord to load video from base64 (#11492)

Browse Source 
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung 2024-12-26 00:33:55 +08:00 committed by GitHub
parent fc601665eb
commit b689ada91e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 19 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
vllm/multimodal/utils.py
View File

@ -125,17 +125,7 @@ async def async_fetch_image(image_url: str,
return image.convert(image_mode) return image.convert(image_mode)
def _load_video_frames_from_bytes(b: bytes): def _load_video_from_bytes(b: bytes, num_frames: int = 32) -> npt.NDArray:
frame = Image.open(BytesIO(b))
return np.array(frame)
def load_video_frames_from_base64(frame: Union[bytes, str]):
"""Load frame from base64 format."""
return _load_video_frames_from_bytes(base64.b64decode(frame))
def _load_video_from_bytes(b: bytes, num_frames: int = 32):
_, decord = try_import_video_packages() _, decord = try_import_video_packages()
video_path = BytesIO(b) video_path = BytesIO(b)
@ -155,13 +145,17 @@ def _load_video_from_bytes(b: bytes, num_frames: int = 32):
return frames return frames
def _load_video_from_data_url(video_url: str): def _load_video_from_data_url(video_url: str) -> npt.NDArray:
# Only split once and assume the second part is the base64 encoded image # Only split once and assume the second part is the base64 encoded video
frames_base64 = video_url.split(",")[1:] _, video_base64 = video_url.split(",", 1)
return np.stack([
load_video_frames_from_base64(frame_base64) if video_url.startswith("data:video/jpeg;"):
for frame_base64 in frames_base64 return np.stack([
]) np.array(load_image_from_base64(frame_base64))
for frame_base64 in video_base64.split(",")
])
return load_video_from_base64(video_base64)
def fetch_video(video_url: str, *, num_frames: int = 32) -> npt.NDArray: def fetch_video(video_url: str, *, num_frames: int = 32) -> npt.NDArray:
@ -342,7 +336,7 @@ def rescale_image_size(image: Image.Image,
return image return image
def try_import_video_packages() -> Any: def try_import_video_packages():
try: try:
import cv2 import cv2
import decord import decord
@ -384,7 +378,7 @@ def sample_frames_from_video(frames: npt.NDArray,
return sampled_frames return sampled_frames
def encode_video_base64(frames: npt.NDArray): def encode_video_base64(frames: npt.NDArray) -> str:
base64_frames = [] base64_frames = []
frames_list = [frames[i] for i in range(frames.shape[0])] frames_list = [frames[i] for i in range(frames.shape[0])]
for frame in frames_list: for frame in frames_list:
@ -393,6 +387,11 @@ def encode_video_base64(frames: npt.NDArray):
return ",".join(base64_frames) return ",".join(base64_frames)
def load_video_from_base64(video: Union[bytes, str]) -> npt.NDArray:
"""Load video from base64 format."""
return _load_video_from_bytes(base64.b64decode(video))
def resolve_visual_encoder_outputs( def resolve_visual_encoder_outputs(
encoder_outputs: Union[torch.Tensor, list[torch.Tensor]], encoder_outputs: Union[torch.Tensor, list[torch.Tensor]],
feature_sample_layers: Optional[list[int]], feature_sample_layers: Optional[list[int]],