sudo apt update
sudo apt upgrade -y
sudo apt install pipx net-tools -y
pipx install huggingface_hub
pipx ensurepath
sudo reboot

cd /home/ubuntu/
hf download Qwen/Qwen3-VL-8B-Thinking-FP8 --local-dir ./Qwen3-VL-8B-Thinking-FP8

cd /home/ubuntu/
curl -LsSf https://astral.sh/uv/install.sh | sh
uv venv --python 3.12 --clear --seed
source .venv/bin/activate
uv pip install vllm bitsandbytes qwen-vl-utils --torch-backend=auto

uv run vllm serve /home/ubuntu/Qwen3-VL-8B-Thinking-FP8 \
    --served-model-name Qwen3-VL-8B \
    --api-key qwen-by-vllm-api-key-2025-11 \
    --gpu-memory-utilization 0.9 \
    --async-scheduling \
    --mm-encoder-tp-mode data \
    --max-model-len 4096 \
    --max-num-seqs 8 \
    --host 0.0.0.0 \
    --port 8080

# vLLM 系统服务自动配置脚本
echo "正在配置 vLLM 系统服务..."

# 创建 systemd 服务文件
sudo tee /etc/systemd/system/vllm.service > /dev/null <<EOF
[Unit]
Description=vLLM Inference Server
After=network.target

[Service]
Type=simple
User=ubuntu
WorkingDirectory=/home/ubuntu
Environment="PATH=/home/ubuntu/.venv/bin:/home/ubuntu/.cargo/bin:/usr/local/bin:/usr/bin"
ExecStart=/home/ubuntu/.cargo/bin/uv run vllm serve /home/ubuntu/Qwen3-VL-8B-Thinking-FP8 \
  --served-model-name Qwen3-VL-8B \
  --api-key qwen-by-vllm-api-key-2025-11 \
  --gpu-memory-utilization 0.9 \
  --async-scheduling \
  --mm-encoder-tp-mode data \
  --max-model-len 4096 \
  --max-num-seqs 8 \
  --host 0.0.0.0 \
  --port 8080
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

echo "服务文件已创建: /etc/systemd/system/vllm.service"

# 重载 systemd 配置
sudo systemctl daemon-reload
echo "systemd 配置已重载"

# 启用开机自启
sudo systemctl enable vllm
echo "已启用开机自启动"

# 重启服务
sudo systemctl restart vllm
echo "服务已重启"

# 等待 5 秒
sleep 5

# 显示服务状态
echo ""
echo "========== 服务状态 =========="
sudo systemctl status vllm --no-pager

echo ""
echo "========== 配置完成 =========="
echo "服务地址: http://0.0.0.0:8080"
echo "模型名称: Qwen3-VL-8B"
echo ""
echo "常用命令:"
echo "  查看状态: sudo systemctl status vllm"
echo "  查看日志: sudo journalctl -u vllm -f"
echo "  停止服务: sudo systemctl stop vllm"
echo "  重启服务: sudo systemctl restart vllm"
echo "  禁用自启: sudo systemctl disable vllm"

curl -LsSf https://astral.sh/uv/install.sh | sh
uv venv --python 3.13
source .venv/bin/activate
uv pip install openai

from openai import OpenAI

client = OpenAI(
    base_url="http://44.247.29.89:8080/v1", 
    # api_key="xxxxxxxxxxxxxx" # 上一步在配置文件中指定的API KEY，这里不推荐hard-code方式，最好是设置OS环境变量
)

stream = client.chat.completions.create(
    model="Qwen3-VL-8B",
    messages=[
        {
            "role": "user",
            "content": "你是什么模型，你有内置的tool能力吗。"
        }
    ],
    stream=True
)

for chunk in stream:
    if chunk.choices[0].delta.content is not None:
        print(chunk.choices[0].delta.content, end="", flush=True)
print()

import base64
from openai import OpenAI

# 读取图片并转换为base64
def encode_image(image_path):
    with open(image_path, "rb") as image_file:
        return base64.b64encode(image_file.read()).decode('utf-8')
# 编码图片
image_base64 = encode_image("image.jpg")

client = OpenAI(
    base_url="http://44.247.29.89:8080/v1", 
    # api_key="xxxxxxxxxxxxxx" # 上一步在配置文件中指定的API KEY，这里不推荐hard-code方式，最好是设置OS环境变量
)

stream = client.chat.completions.create(
    model="Qwen3-VL-8B",
    messages=[
        {
            "role": "user",
            "content": [
                {
                    "type": "text",
                    "text": "告诉我图中发生了什么"
                },
                {
                    "type": "image_url",
                    "image_url": {
                        "url": f"data:image/jpeg;base64,{image_base64}"
                    }
                }
            ]
        }
    ],
    stream=True
)

for chunk in stream:
    if chunk.choices[0].delta.content is not None:
        print(chunk.choices[0].delta.content, end="", flush=True)
print()

uv run vllm bench throughput \
    --model /home/ubuntu/Qwen3-VL-8B-Thinking-FP8 \
    --gpu-memory-utilization 0.9 \
    --max-model-len 4096 \
    --max-num-seqs 32 \
    --num-prompts 50 \
    --input-len 512 \
    --output-len 256

Every 1.0s: nvidia-smi                                                         ip-172-31-1-204: Fri Nov 28 10:36:56 2025

Fri Nov 28 10:36:56 2025
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 580.95.05              Driver Version: 580.95.05      CUDA Version: 13.0     |
+-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA L4                      On  |   00000000:31:00.0 Off |                    0 |
| N/A   68C    P0             71W /   72W |   14954MiB /  23034MiB |     97%      Default |
|                                         |                        |                  N/A |
+-----------------------------------------+------------------------+----------------------+

+-----------------------------------------------------------------------------------------+
| Processes:                                                                              |
|  GPU   GI   CI              PID   Type   Process name                        GPU Memory |
|        ID   ID                                                               Usage      |
|=========================================================================================|
|    0   N/A  N/A           14831      C   VLLM::EngineCore                      14946MiB |
+-----------------------------------------------------------------------------------------+

Processed prompts: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:12<00:00,  4.15it/s, est. speed input: 2127.26 toks/s, output: 1063.63 toks/s]
[rank0]:[W1129 06:04:03.381717598 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
Throughput: 4.12 requests/s, 3165.21 total tokens/s, 1055.07 output tokens/s
Total num prompt tokens:  25600

Processed prompts: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:07<00:00,  6.44it/s, est. speed input: 3298.99 toks/s, output: 1649.49 toks/s]
[rank0]:[W1129 06:00:29.273824291 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
Throughput: 6.36 requests/s, 4883.75 total tokens/s, 1627.92 output tokens/s
Total num prompt tokens:  25600
Total num output tokens:  12800

Processed prompts: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:07<00:00,  6.44it/s, est. speed input: 3297.59 toks/s, output: 1648.79 toks/s]
[rank0]:[W1129 07:15:44.201402217 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
Throughput: 6.37 requests/s, 4890.88 total tokens/s, 1630.29 output tokens/s
Total num prompt tokens:  25600
Total num output tokens:  12800

uv run vllm bench latency \
    --model /home/ubuntu/Qwen3-VL-8B-Thinking-FP8 \
    --gpu-memory-utilization 0.9 \
    --max-model-len 4096 \
    --max-num-seqs 32 \
    --input-len 512 \
    --output-len 256

Avg latency: 11.907208129666635 seconds
10% percentile latency: 11.855494655300095 seconds
25% percentile latency: 11.87158773074998 seconds
50% percentile latency: 11.905295175499987 seconds
75% percentile latency: 11.951438236249999 seconds
90% percentile latency: 11.96355620340007 seconds
99% percentile latency: 11.977924459430096 seconds

Avg latency: 4.861004937666606 seconds
10% percentile latency: 4.853756664999674 seconds
25% percentile latency: 4.854472121999947 seconds
50% percentile latency: 4.8599380749999455 seconds
75% percentile latency: 4.86551012974985 seconds
90% percentile latency: 4.870080923500245 seconds
99% percentile latency: 4.877819421329982 seconds
[rank0]:[W1129 05:46:08.955282032 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())