Production Deployment

version: '3.8'
services:
  vllm:
    image: vllm/vllm-openai-rocm:latest
    container_name: vllm-server
    ipc: host
    devices:
      - /dev/kfd
      - /dev/dri
    group_add:
      - video
    security_opt:
      - seccomp:unconfined
    cap_add:
      - SYS_PTRACE
    ports:
      - "8000:8000"
    environment:
      - HF_TOKEN=${HF_TOKEN}
      - VLLM_ROCM_USE_AITER=1
      - AITER_ENABLE_VSKIP=0
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    command: >
      --model deepseek-ai/DeepSeek-V3.2
      --tensor-parallel-size 8
      --quantization fp8
      --block-size 1
      --host 0.0.0.0
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 600s

# Basic health check
curl http://localhost:8000/health

# Verify model is loaded and ready
curl http://localhost:8000/v1/models

scrape_configs:
  - job_name: 'vllm'
    scrape_interval: 15s
    static_configs:
      - targets: ['vllm-server:8000']
    metrics_path: /metrics

curl http://localhost:8000/metrics

version: '3.8'
services:
  vllm:
    image: vllm/vllm-openai-rocm:latest
    container_name: vllm-server
    ipc: host
    devices:
      - /dev/kfd
      - /dev/dri
    group_add:
      - video
    security_opt:
      - seccomp:unconfined
    cap_add:
      - SYS_PTRACE
    ports:
      - "8000:8000"
    environment:
      - HF_TOKEN=${HF_TOKEN}
      - VLLM_ROCM_USE_AITER=1
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    command: >
      --model meta-llama/Llama-3.1-405B-Instruct
      --tensor-parallel-size 8
      --quantization fp8
      --max-model-len 32768
      --host 0.0.0.0
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 600s

  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
    restart: unless-stopped

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    volumes:
      - grafana_data:/var/lib/grafana
    restart: unless-stopped

volumes:
  prometheus_data:
  grafana_data:

global:
  scrape_interval: 15s

scrape_configs:
  - job_name: 'vllm'
    static_configs:
      - targets: ['vllm-server:8000']
    metrics_path: /metrics

# Follow logs
docker logs -f vllm-server

# Last 100 lines
docker logs --tail 100 vllm-server

services:
  vllm:
    # ... other config ...
    logging:
      driver: json-file
      options:
        max-size: "100m"
        max-file: "3"

services:
  vllm:
    # ... other config ...
    stop_grace_period: 30s

services:
  nginx:
    image: nginx:latest
    ports:
      - "80:80"
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf
    depends_on:
      - vllm-1
      - vllm-2

upstream vllm {
    least_conn;
    server vllm-1:8000;
    server vllm-2:8000;
}

server {
    listen 80;

    location / {
        proxy_pass http://vllm;
        proxy_http_version 1.1;
        proxy_set_header Connection "";
        proxy_connect_timeout 300s;
        proxy_read_timeout 300s;
    }

    location /health {
        proxy_pass http://vllm;
    }
}

groups:
  - name: vllm
    rules:
      - alert: VLLMHighLatency
        expr: histogram_quantile(0.95, rate(vllm:e2e_request_latency_seconds_bucket[5m])) > 30
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High p95 latency on vLLM"

      - alert: VLLMHighQueueDepth
        expr: vllm:num_requests_waiting > 100
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "High queue depth on vLLM"

      - alert: VLLMDown
        expr: up{job="vllm"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "vLLM server is down"