LiteLLM으로 vLLM 앞단 프록시 구축하기

LITELLM=~/.local/share/uv/tools/litellm PATH="$LITELLM/bin:$PATH" "$LITELLM/bin/python" -m prisma generate \
 --schema="$LITELLM/lib/python3.12/site-packages/litellm_proxy_extras/schema.prisma"

model_list:
- model_name: glm-5.2
  litellm_params:
    model: openai/glm-5.2
    api_base: http://:8000/v1
    api_key:
    stream_timeout: 900
  model_info:
    health_check_timeout: 5
    input_cost_per_token: 0.00000035
    output_cost_per_token: 0.00000175
- model_name: glm-5.2 # same name, second backend
...
general_settings:
  background_health_checks: true
  health_check_interval: 30
  enable_health_check_routing: true
  health_check_ignore_transient_errors: true
router_settings:
  routing_strategy: simple-shuffle
  enable_weighted_failover: true
  num_retries: 2
  retry_after: 1
  timeout: 5
  cooldown_time: 60
  allowed_fails_policy:
    AuthenticationErrorAllowedFails: 0
    TimeoutErrorAllowedFails: 2
    RateLimitErrorAllowedFails: 5

[Service]
User=litellm
EnvironmentFile=/home/litellm/.config/litellm/litellm.env
Environment="PATH=/home/litellm/.local/bin:/usr/local/bin:/usr/bin:/bin"
ExecStart=/home/litellm/.local/bin/litellm --config /home/litellm/.config/litellm/config.yml
NoNewPrivileges=true
ProtectSystem=strict
ReadWritePaths=/home/litellm
PrivateTmp=true
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
LockPersonality=true
RestrictSUIDSGID=true
CapabilityBoundingSet=
Restart=on-failure
RestartSec=5s
TimeoutStartSec=180

location /v1 {
 proxy_pass http://litellm_backend;
 proxy_http_version 1.1;
 proxy_set_header Connection "";
 proxy_set_header Host $host;
 proxy_set_header X-Real-IP $remote_addr;
 proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 proxy_buffering off;
 proxy_request_buffering off;
 proxy_cache off;
 gzip off;
 proxy_connect_timeout 60s;
 proxy_send_timeout 1800s;
 proxy_read_timeout 1800s;
 proxy_next_upstream off;
 proxy_intercept_errors off;
 send_timeout 1800s;
 reset_timedout_connection on;
 lingering_close always;
 lingering_timeout 30s;
}

proxy_hide_header x-litellm-model-api-base;
proxy_hide_header x-litellm-model-id;
proxy_hide_header x-litellm-model-name;
proxy_hide_header x-litellm-response-cost;
proxy_hide_header x-litellm-key-spend;
proxy_hide_header x-litellm-key-max-budget;
# ... the rest of the x-litellm-* family
location = /openapi.json { return 404; }
location ^~ /swagger { return 404; }
location = /redoc { return 404; }
location ^~ /docs { return 404; }
location = / { return 404; }