run.sh

#!/bin/sh


printf "Running vLLM OpenAI compatible API Server at port %s\n" "7860"

#python -u /app/openai_compatible_api_server.py \
#    --model meta-llama/Llama-3.2-3B-Instruct \
#    --revision 0cb88a4f764b7a12671c53f0838cd831a0843b95 \
#    --host 0.0.0.0 \
#    --port 7860 \
#    --max-num-batched-tokens 32768 \
#    --max-model-len 32768 \
#    --dtype half \
#    --enforce-eager \
#    --gpu-memory-utilization 0.85


# Reducing max-num-batched-tokens to 7536 because got this error:
# INFO 11-27 15:32:01 model_runner.py:1077] Loading model weights took 7.4150 GB
# INFO 11-27 15:32:09 worker.py:232] Memory profiling results: total_gpu_memory=14.58GiB initial_memory_usage=7.61GiB peak_torch_memory=9.31GiB memory_usage_post_profile=7.62GiB non_torch_memory=0.20GiB kv_cache_size=2.88GiB gpu_memory_utilization=0.85
# INFO 11-27 15:32:10 gpu_executor.py:113] # GPU blocks: 471, # CPU blocks: 655
# INFO 11-27 15:32:10 gpu_executor.py:117] Maximum concurrency for 32768 tokens per request: 0.23x
# ERROR 11-27 15:32:10 engine.py:366] The model's max seq len (32768) is larger than the maximum number of tokens that can be stored in KV cache (7536). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine.
python -u /app/openai_compatible_api_server.py \
    --model sail/Sailor-4B-Chat \
    --revision 89a866a7041e6ec023dd462adeca8e28dd53c83e \
    --host 0.0.0.0 \
    --port 7860 \
    --max-num-batched-tokens 7536 \
    --dtype half \
    --enforce-eager \
    --gpu-memory-utilization 0.85