vLLM Cheatsheet

# docker-compose.yml
services:
  vllm:
    image: vllm/vllm-openai:latest
    runtime: nvidia
    ports:
      - "8000:8000"
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    environment:
      - HF_TOKEN=${HF_TOKEN}
      - CUDA_VISIBLE_DEVICES=0,1
    ipc: host
    command: >
      --model meta-llama/Llama-3.1-70B-Instruct
      --tensor-parallel-size 2
      --gpu-memory-utilization 0.90
      --max-model-len 8192
      --api-key ${VLLM_API_KEY}
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 2
              capabilities: [gpu]
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 120s
    restart: unless-stopped

# /etc/systemd/system/vllm.service
[Unit]
Description=vLLM OpenAI-compatible API Server
After=network.target docker.service
Requires=docker.service

[Service]
Type=simple
ExecStart=/usr/bin/docker compose -f /opt/vllm/docker-compose.yml up
ExecStop=/usr/bin/docker compose -f /opt/vllm/docker-compose.yml down
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target

apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm
spec:
  replicas: 1
  selector:
    matchLabels:
      app: vllm
  template:
    metadata:
      labels:
        app: vllm
    spec:
      containers:
        - name: vllm
          image: vllm/vllm-openai:latest
          args:
            - "--model"
            - "meta-llama/Llama-3.1-8B-Instruct"
            - "--gpu-memory-utilization"
            - "0.90"
          ports:
            - containerPort: 8000
          resources:
            limits:
              nvidia.com/gpu: 1
          livenessProbe:
            httpGet:
              path: /health
              port: 8000
            initialDelaySeconds: 120
            periodSeconds: 30
          readinessProbe:
            httpGet:
              path: /health
              port: 8000
            initialDelaySeconds: 60
            periodSeconds: 10
          env:
            - name: HF_TOKEN
              valueFrom:
                secretKeyRef:
                  name: hf-token
                  key: token
          volumeMounts:
            - name: model-cache
              mountPath: /root/.cache/huggingface
      volumes:
        - name: model-cache
          persistentVolumeClaim:
            claimName: model-cache-pvc
---
apiVersion: v1
kind: Service
metadata:
  name: vllm
  annotations:
    prometheus.io/scrape: "true"
    prometheus.io/port: "8000"
    prometheus.io/path: "/metrics"
spec:
  selector:
    app: vllm
  ports:
    - port: 8000
      targetPort: 8000
  type: ClusterIP

# vLLM exposes Prometheus metrics at /metrics
# Key metrics to monitor:

# Request throughput
vllm:num_requests_running          # Currently running requests
vllm:num_requests_waiting          # Queued requests
vllm:request_success_total         # Total successful requests

# Latency
vllm:e2e_request_latency_seconds   # End-to-end latency histogram
vllm:time_to_first_token_seconds   # Time to first token (TTFT)
vllm:time_per_output_token_seconds # Time per output token (TPOT)

# GPU utilization
vllm:gpu_cache_usage_perc          # KV cache utilization
vllm:cpu_cache_usage_perc          # CPU swap cache utilization

# Model info
vllm:num_preemptions_total         # Preempted requests (VRAM pressure)
vllm:avg_generation_throughput_toks_per_s  # Tokens/sec

# Scrape config for Prometheus
scrape_configs:
  - job_name: 'vllm'
    static_configs:
      - targets: ['vllm-host:8000']

# Warmup script — run after server starts to JIT compile CUDA kernels
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{"model": "llama-3.1", "messages": [{"role": "user", "content": "Hello"}], "max_tokens": 1}'

# Or in Python
import requests
for _ in range(3):
    requests.post("http://localhost:8000/v1/chat/completions", json={
        "model": "llama-3.1",
        "messages": [{"role": "user", "content": "warmup"}],
        "max_tokens": 1
    })