Kubernetes에서 LLM 서빙하기: vLLM, TGI, Triton 완전 가이드

flowchart TB
    Client["클라이언트 요청들"] --> LB[Load Balancer]
    LB --> Pod1["vLLM Pod<br/>GPU A100 x2"]
    LB --> Pod2["vLLM Pod<br/>GPU A100 x2"]
    LB --> Pod3["vLLM Pod<br/>GPU A100 x2"]
    HPA[HPA<br/>자동 스케일링] --> Pod1
    HPA --> Pod2
    HPA --> Pod3
    Pod1 --> Model[(모델 스토리지<br/>PVC/NFS)]
    Pod2 --> Model
    Pod3 --> Model

# vllm-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-server
spec:
  replicas: 2
  selector:
    matchLabels:
      app: vllm
  template:
    metadata:
      labels:
        app: vllm
    spec:
      containers:
      - name: vllm
        image: vllm/vllm-openai:latest
        args:
        - "--model"
        - "meta-llama/Llama-3.1-8B-Instruct"
        - "--tensor-parallel-size"
        - "2"              # GPU 2개 병렬 사용
        - "--max-model-len"
        - "8192"
        - "--gpu-memory-utilization"
        - "0.90"
        - "--enable-chunked-prefill"  # 긴 프롬프트 청크 처리
        ports:
        - containerPort: 8000
        resources:
          limits:
            nvidia.com/gpu: "2"
            memory: "80Gi"
          requests:
            nvidia.com/gpu: "2"
            memory: "60Gi"
        env:
        - name: HUGGING_FACE_HUB_TOKEN
          valueFrom:
            secretKeyRef:
              name: hf-token
              key: token
        volumeMounts:
        - name: model-cache
          mountPath: /root/.cache/huggingface
      volumes:
      - name: model-cache
        persistentVolumeClaim:
          claimName: model-cache-pvc
      tolerations:
      - key: "nvidia.com/gpu"
        operator: "Exists"
        effect: "NoSchedule"

from openai import OpenAI

# vLLM은 OpenAI API와 호환
client = OpenAI(
    base_url="http://vllm-service:8000/v1",
    api_key="token-abc123"
)

response = client.chat.completions.create(
    model="meta-llama/Llama-3.1-8B-Instruct",
    messages=[{"role": "user", "content": "안녕하세요!"}],
    max_tokens=512,
    temperature=0.7,
    stream=True  # 스트리밍 지원
)

for chunk in response:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="")

# tgi-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tgi-server
spec:
  replicas: 1
  template:
    spec:
      containers:
      - name: tgi
        image: ghcr.io/huggingface/text-generation-inference:latest
        args:
        - "--model-id"
        - "mistralai/Mistral-7B-Instruct-v0.3"
        - "--num-shard"
        - "1"
        - "--max-concurrent-requests"
        - "128"
        - "--max-batch-prefill-tokens"
        - "4096"
        - "--quantize"
        - "bitsandbytes-nf4"  # 4비트 양자화
        ports:
        - containerPort: 80
        resources:
          limits:
            nvidia.com/gpu: "1"

# keda-scaledobject.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: vllm-scaler
spec:
  scaleTargetRef:
    name: vllm-server
  minReplicaCount: 1
  maxReplicaCount: 8
  cooldownPeriod: 300  # GPU 워밍업 고려
  triggers:
  - type: prometheus
    metadata:
      serverAddress: http://prometheus:9090
      metricName: vllm_requests_waiting
      threshold: "10"  # 대기 요청 10개 초과 시 스케일업
      query: sum(vllm:num_requests_waiting)

initContainers:
- name: model-downloader
  image: python:3.11-slim
  command:
  - python
  - -c
  - |
    from huggingface_hub import snapshot_download
    snapshot_download(
        repo_id="meta-llama/Llama-3.1-8B-Instruct",
        local_dir="/models/llama-3.1-8b",
        ignore_patterns=["*.msgpack", "*.h5"]
    )
  volumeMounts:
  - name: model-storage
    mountPath: /models
  env:
  - name: HUGGING_FACE_HUB_TOKEN
    valueFrom:
      secretKeyRef:
        name: hf-token
        key: token