2025-01-13 12:27:36 +00:00
|
|
|
# GPU
|
|
|
|
|
|
|
|
|
|
vLLM is a Python library that supports the following GPU variants. Select your GPU type to see vendor specific instructions:
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::{tab-set}
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync-group: device
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} NVIDIA CUDA
|
|
|
|
|
:selected:
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: cuda
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} cuda.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "# Installation"
|
|
|
|
|
:end-before: "## Requirements"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} AMD ROCm
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: rocm
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} rocm.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "# Installation"
|
|
|
|
|
:end-before: "## Requirements"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} Intel XPU
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: xpu
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} xpu.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "# Installation"
|
|
|
|
|
:end-before: "## Requirements"
|
|
|
|
|
:::
|
|
|
|
|
|
|
|
|
|
::::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::
|
|
|
|
|
|
2025-01-13 12:27:36 +00:00
|
|
|
## Requirements
|
|
|
|
|
|
|
|
|
|
- OS: Linux
|
|
|
|
|
- Python: 3.9 -- 3.12
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::{tab-set}
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync-group: device
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} NVIDIA CUDA
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: cuda
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} cuda.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "## Requirements"
|
|
|
|
|
:end-before: "## Set up using Python"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} AMD ROCm
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: rocm
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} rocm.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "## Requirements"
|
|
|
|
|
:end-before: "## Set up using Python"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} Intel XPU
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: xpu
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} xpu.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "## Requirements"
|
|
|
|
|
:end-before: "## Set up using Python"
|
|
|
|
|
:::
|
|
|
|
|
|
|
|
|
|
::::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::
|
|
|
|
|
|
2025-01-13 12:27:36 +00:00
|
|
|
## Set up using Python
|
|
|
|
|
|
|
|
|
|
### Create a new Python environment
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} ../python_env_setup.inc.md
|
|
|
|
|
:::
|
2025-01-13 12:27:36 +00:00
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::{tab-set}
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync-group: device
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} NVIDIA CUDA
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: cuda
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} cuda.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "## Create a new Python environment"
|
|
|
|
|
:end-before: "### Pre-built wheels"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} AMD ROCm
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: rocm
|
|
|
|
|
|
|
|
|
|
There is no extra information on creating a new Python environment for this device.
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
2025-01-13 12:27:36 +00:00
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} Intel XPU
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: xpu
|
|
|
|
|
|
|
|
|
|
There is no extra information on creating a new Python environment for this device.
|
|
|
|
|
|
|
|
|
|
::::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::
|
|
|
|
|
|
2025-01-13 12:27:36 +00:00
|
|
|
### Pre-built wheels
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::{tab-set}
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync-group: device
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} NVIDIA CUDA
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: cuda
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} cuda.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "### Pre-built wheels"
|
|
|
|
|
:end-before: "### Build wheel from source"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} AMD ROCm
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: rocm
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} rocm.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "### Pre-built wheels"
|
|
|
|
|
:end-before: "### Build wheel from source"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} Intel XPU
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: xpu
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} xpu.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "### Pre-built wheels"
|
|
|
|
|
:end-before: "### Build wheel from source"
|
|
|
|
|
:::
|
|
|
|
|
|
|
|
|
|
::::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::
|
|
|
|
|
|
2025-01-13 12:27:36 +00:00
|
|
|
(build-from-source)=
|
|
|
|
|
|
|
|
|
|
### Build wheel from source
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::{tab-set}
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync-group: device
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} NVIDIA CUDA
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: cuda
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} cuda.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "### Build wheel from source"
|
|
|
|
|
:end-before: "## Set up using Docker"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} AMD ROCm
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: rocm
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} rocm.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "### Build wheel from source"
|
|
|
|
|
:end-before: "## Set up using Docker"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} Intel XPU
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: xpu
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} xpu.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "### Build wheel from source"
|
|
|
|
|
:end-before: "## Set up using Docker"
|
|
|
|
|
:::
|
|
|
|
|
|
|
|
|
|
::::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::
|
|
|
|
|
|
2025-01-13 12:27:36 +00:00
|
|
|
## Set up using Docker
|
|
|
|
|
|
|
|
|
|
### Pre-built images
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::{tab-set}
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync-group: device
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} NVIDIA CUDA
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: cuda
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} cuda.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "### Pre-built images"
|
|
|
|
|
:end-before: "### Build image from source"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} AMD ROCm
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: rocm
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} rocm.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "### Pre-built images"
|
|
|
|
|
:end-before: "### Build image from source"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} Intel XPU
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: xpu
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} xpu.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "### Pre-built images"
|
|
|
|
|
:end-before: "### Build image from source"
|
|
|
|
|
:::
|
|
|
|
|
|
|
|
|
|
::::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::
|
|
|
|
|
|
2025-01-13 12:27:36 +00:00
|
|
|
### Build image from source
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::{tab-set}
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync-group: device
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} NVIDIA CUDA
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: cuda
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} cuda.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "### Build image from source"
|
|
|
|
|
:end-before: "## Supported features"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} AMD ROCm
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: rocm
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} rocm.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "### Build image from source"
|
|
|
|
|
:end-before: "## Supported features"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} Intel XPU
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: xpu
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} xpu.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "### Build image from source"
|
|
|
|
|
:end-before: "## Supported features"
|
|
|
|
|
:::
|
|
|
|
|
|
|
|
|
|
::::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::
|
|
|
|
|
|
2025-01-13 12:27:36 +00:00
|
|
|
## Supported features
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::::{tab-set}
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync-group: device
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} NVIDIA CUDA
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: cuda
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} cuda.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "## Supported features"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} AMD ROCm
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: rocm
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} rocm.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "## Supported features"
|
|
|
|
|
:::
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
::::
|
|
|
|
|
|
2025-01-31 23:38:35 +00:00
|
|
|
::::{tab-item} Intel XPU
|
2025-01-13 12:27:36 +00:00
|
|
|
:sync: xpu
|
|
|
|
|
|
2025-01-29 03:38:29 +00:00
|
|
|
:::{include} xpu.inc.md
|
2025-01-13 12:27:36 +00:00
|
|
|
:start-after: "## Supported features"
|
|
|
|
|
:::
|
|
|
|
|
|
|
|
|
|
::::
|
2025-01-29 03:38:29 +00:00
|
|
|
|
|
|
|
|
:::::
|
