runner

1# LiteLLM Proxy Configuration
2# Maps Ollama models to OpenAI-compatible API endpoints
3#
4# `provider` per model (llm_stack_models): ollama by default, ollama_chat for
5# vision models. The `ollama/` route converts images with Pillow, which the
6# LiteLLM image does not ship, so every image request through it fails with a
7# 500 ("ollama image conversion failed").
8
9model_list:
10{% for model in llm_stack_models %}
11  - model_name: "{{ model.name }}"
12    litellm_params:
13      model: "{{ model.provider | default('ollama') }}/{{ model.name }}"
14      api_base: "http://ollama:11434"
15      stream: {{ llm_stack_litellm_streaming_enabled | lower }}
16      # Cut a hung upstream call here rather than holding the request open.
17      # Longer than the pipeline's own 900s client timeout, so the client
18      # remains the authority; this only stops litellm waiting forever when a
19      # caller has no timeout of its own.
20      request_timeout: 1200
21{% endfor %}
22
23litellm_settings:
24  # **Discards provider-specific keys litellm does not recognise**, which is
25  # right for a proxy that normalises across providers -- a stray key is a 400
26  # from somewhere else.
27  #
28  # > ⚠️ It drops a nested `options` block, so `{"options":{"num_ctx":4096}}`
29  # > never reaches Ollama. An unknown *top-level* key IS passed through, so
30  # > `{"num_ctx":4096}` works. Measured 2026-09-01 against this proxy, same
31  # > endpoint and payload otherwise: nested -> loaded at 24576 and 11 GB;
32  # > top-level -> 4096 and 9.3 GB. The live judge sends it top-level
33  # > (`dnd_session/llm.py`), which is what keeps the 14b beside Whisper on a
34  # > 12 GB card.
35  drop_params: true
36  set_verbose: {{ true if llm_stack_litellm_log_level == 'debug' else false }}
37  # Caching off, deliberately. The pipeline's prompts are unique per call, so
38  # the cache buys nothing -- and the cache-write path sits exactly where a
39  # hung request was observed (accepted, never answered). The pipeline's own
40  # determinism concerns are handled at its level, not by caching behind it.
41  cache: false
42
43general_settings:
44  master_key: "{{ vault_runner.lite_llm_master_key | default('sk-1234') }}"
45  database_url: "postgresql://litellm:{{ vault_runner.lite_llm_db_password | default('litellm123') }}@postgres:5432/litellm"
46  store_model_in_db: true
47