2023-06-14 19:55:38 -07:00
|
|
|
#!/bin/bash
|
|
|
|
|
|
2023-06-26 13:15:35 -07:00
|
|
|
PORT=8000
|
2023-06-14 19:55:38 -07:00
|
|
|
MODEL=$1
|
|
|
|
|
TOKENS=$2
|
|
|
|
|
|
2024-11-07 13:17:29 -05:00
|
|
|
docker run -e "HF_TOKEN=$HF_TOKEN" --gpus all --shm-size 1g -p $PORT:80 \
|
|
|
|
|
-v "$PWD/data:/data" \
|
2024-08-28 00:07:53 +02:00
|
|
|
ghcr.io/huggingface/text-generation-inference:2.2.0 \
|
2024-11-07 13:17:29 -05:00
|
|
|
--model-id "$MODEL" \
|
2023-06-14 19:55:38 -07:00
|
|
|
--sharded false \
|
|
|
|
|
--max-input-length 1024 \
|
|
|
|
|
--max-total-tokens 2048 \
|
|
|
|
|
--max-best-of 5 \
|
|
|
|
|
--max-concurrent-requests 5000 \
|
2024-11-07 13:17:29 -05:00
|
|
|
--max-batch-total-tokens "$TOKENS"
|
