20231088/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CI/Build] Adding functionality to reset the node's GPUs before processing. (#4213)

Browse Source
This commit is contained in:
Alexei-V-Ivanov-AMD 2024-04-25 11:37:20 -05:00 committed by GitHub
parent fbf152d976
commit 7ee82bef1e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 15 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.buildkite/run-amd-test.sh
View File

@ -5,6 +5,19 @@ set -ex
# Print ROCm version # Print ROCm version
rocminfo rocminfo
echo "reset" > /opt/amdgpu/etc/gpu_state
while true; do
sleep 3
if grep -q clean /opt/amdgpu/etc/gpu_state; then
echo "GPUs state is \"clean\""
break
fi
done
# Try building the docker image # Try building the docker image
docker build -t rocm -f Dockerfile.rocm . docker build -t rocm -f Dockerfile.rocm .
@ -14,7 +27,8 @@ trap remove_docker_container EXIT
remove_docker_container remove_docker_container
# Run the image # Run the image
docker run --device /dev/kfd --device /dev/dri --network host --name rocm rocm python3 -m vllm.entrypoints.api_server & export HIP_VISIBLE_DEVICES=1
docker run --device /dev/kfd --device /dev/dri --network host -e HIP_VISIBLE_DEVICES --name rocm rocm python3 -m vllm.entrypoints.api_server &
# Wait for the server to start # Wait for the server to start
wait_for_server_to_start() { wait_for_server_to_start() {